Upload model - 26kn31z5

README.md

@@ -43,7 +43,6 @@ Data is prepared with the `ocf_data_sampler/torch_datasets/datasets/pvnet_uk` Da
 
 <!-- Do not remove the lines below -->
 The training logs for this model commit can be found here:
- - [https://wandb.ai/openclimatefix/NL-Solar/runs/o9kjxito](https://wandb.ai/openclimatefix/NL-Solar/runs/o9kjxito)
 
 
 <!-- The hardware section is also just nice to have -->
@@ -60,5 +59,5 @@ This model was trained using the following Open Climate Fix packages:
 
 <!-- Especially do not change the two lines below -->
 The versions of these packages can be found below:
- - pvnet==5.1.3.post2+git.fe7c3933.dirty
+ - pvnet_summation==1.1.0.post0+git.3b6f1115.dirty
  - ocf-data-sampler==0.6.0.post0+git.a508745b.dirty

datamodule_config.yaml

@@ -0,0 +1,3 @@
+pvnet_model:
+  model_id: openclimatefix-models/pvnet_nl
+  revision: de5ad0748be8952a4c0600a1ae6455bf802f5896

full_experiment_config.yaml

@@ -0,0 +1,121 @@
+trainer:
+  _target_: lightning.pytorch.trainer.trainer.Trainer
+  accelerator: gpu
+  devices:
+  - 1
+  min_epochs: null
+  max_epochs: 100
+  reload_dataloaders_every_n_epochs: 0
+  num_sanity_val_steps: 8
+  fast_dev_run: false
+  log_every_n_steps: 50
+model:
+  _target_: pvnet_summation.training.lightning_module.PVNetSummationLightningModule
+  model:
+    _target_: pvnet_summation.models.horizon_dense_model.HorizonDenseModel
+    output_quantiles:
+    - 0.02
+    - 0.1
+    - 0.25
+    - 0.5
+    - 0.75
+    - 0.9
+    - 0.98
+    output_network:
+      _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
+      _partial_: true
+      fc_hidden_features: 32
+      n_res_blocks: 1
+      res_block_layers: 2
+      dropout_frac: 0.2
+    predict_difference_from_sum: true
+    use_horizon_encoding: true
+    use_solar_position: true
+    force_non_crossing: true
+    history_minutes: 2880
+    forecast_minutes: 2160
+    interval_minutes: 15
+    num_input_locations: 12
+    input_quantiles:
+    - 0.02
+    - 0.1
+    - 0.25
+    - 0.5
+    - 0.75
+    - 0.9
+    - 0.98
+  optimizer:
+    _target_: pvnet_summation.optimizers.AdamWReduceLROnPlateau
+    lr: 0.0001
+    weight_decay: 0.25
+    amsgrad: true
+    patience: 10
+    factor: 0.1
+    threshold: 0.0
+datamodule:
+  data_source_paths:
+    site:
+      file_path: /home/alex/NL/NL_regional_generation_kw.nc
+      metadata_file_path: /home/alex/NL/NL_regional_metadata.csv
+    satellite:
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2019_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2020_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2021_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v1/2022_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v3/2023_nonhrv.zarr
+    - /mnt/storage_u2_30tb_a/ml_training_zarrs/sat/uk_sat_crops/v3/2024_nonhrv.zarr
+    nwp:
+      ecmwf: /mnt/storage_u2_4tb_a/ecmwf_nl_crop_rechunked/*_float32.zarr
+  batch_size: 32
+  num_workers: 8
+  prefetch_factor: 2
+  persistent_workers: true
+  max_num_train_samples: null
+  max_num_val_samples: null
+  pvnet_model:
+    model_id: openclimatefix-models/pvnet_nl
+    revision: de5ad0748be8952a4c0600a1ae6455bf802f5896
+  train_period:
+  - null
+  - '2023-12-31'
+  val_period:
+  - '2024-01-01'
+  - '2025-01-01'
+  seed: ${seed}
+  dataset_pickle_dir: null
+callbacks:
+  learning_rate_monitor:
+    _target_: lightning.pytorch.callbacks.LearningRateMonitor
+    logging_interval: epoch
+  model_summary:
+    _target_: lightning.pytorch.callbacks.ModelSummary
+    max_depth: 3
+  model_checkpoint:
+    _target_: lightning.pytorch.callbacks.ModelCheckpoint
+    monitor: ${resolve_monitor_loss:${model.model.output_quantiles}}
+    mode: min
+    save_top_k: 1
+    save_last: true
+    every_n_epochs: 1
+    verbose: false
+    filename: epoch={epoch}-step={step}
+    dirpath: /home/alex/NL/callbakcs/${model_name}
+    auto_insert_metric_name: false
+    save_on_train_epoch_end: false
+logger:
+  wandb:
+    _target_: lightning.pytorch.loggers.wandb.WandbLogger
+    project: NL-Solar
+    name: ${model_name}
+    save_dir: /home/alex/NL/wandb
+    offline: false
+    id: null
+    log_model: false
+    prefix: ''
+    job_type: train
+    group: ''
+    tags: []
+sample_save_dir: /mnt/storage_u2_30tb_a/NL_summation_samples
+work_dir: ${hydra:runtime.cwd}
+model_name: v7_summation_horizon
+seed: 2727831

model_config.yaml

@@ -1,4 +1,4 @@
-_target_: pvnet.models.LateFusionModel
+_target_: pvnet_summation.models.horizon_dense_model.HorizonDenseModel
 output_quantiles:
 - 0.02
 - 0.1
@@ -7,49 +7,26 @@ output_quantiles:
 - 0.75
 - 0.9
 - 0.98
-nwp_encoders_dict:
-  ecmwf:
-    _target_: pvnet.models.late_fusion.encoders.encoders3d.DefaultPVNet
-    _partial_: true
-    in_channels: 13
-    out_features: 64
-    fc_features: 32
-    number_of_conv3d_layers: 4
-    conv3d_channels: 32
-    image_size_pixels: 10
 output_network:
   _target_: pvnet.models.late_fusion.linear_networks.networks.ResFCNet
   _partial_: true
-  fc_hidden_features: 128
-  n_res_blocks: 6
+  fc_hidden_features: 32
+  n_res_blocks: 1
   res_block_layers: 2
   dropout_frac: 0.2
-include_sun: true
-target_key: site
-include_gsp_yield_history: false
-include_site_yield_history: true
-interval_minutes: 15
-location_id_mapping:
-  1: 1
-  2: 2
-  3: 3
-  4: 4
-  5: 5
-  6: 6
-  7: 7
-  8: 8
-  9: 9
-  10: 10
-  11: 11
-  12: 12
-forecast_minutes: 2160
+predict_difference_from_sum: true
+use_horizon_encoding: true
+use_solar_position: true
+force_non_crossing: true
 history_minutes: 2880
-min_sat_delay_minutes: 5
-sat_history_minutes: 60
-pv_history_minutes: 2880
-nwp_history_minutes:
-  ecmwf: 120
-nwp_forecast_minutes:
-  ecmwf: 2220
-nwp_interval_minutes:
-  ecmwf: 60
+forecast_minutes: 2160
+interval_minutes: 15
+num_input_locations: 12
+input_quantiles:
+- 0.02
+- 0.1
+- 0.25
+- 0.5
+- 0.75
+- 0.9
+- 0.98

model_weights.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:beffa7adb39faddc31f7f22be5348b56c3e85a8b9e6353996554ea7d971bf4f3
-size 2581760
+oid sha256:5aa051d135bc8fa1ddbf76a8ed290ea88e8c9ae7836d0f28b89703fca025f4ce
+size 23684