diff --git a/examples/circularity/commodities.csv b/examples/circularity/commodities.csv index 092ca5baa..c8179b6a6 100644 --- a/examples/circularity/commodities.csv +++ b/examples/circularity/commodities.csv @@ -1,13 +1,13 @@ -id,description,type,time_slice_level -OILCRD,Crude oil,sed,season -GASPRD,Gas produced,sed,season -GASOLI,Gasoline,sed,season -DIESEL,Diesel,sed,season -GASNAT,Natural gas,sed,season -ELCTRI,Electricity,sed,daynight -H2YPRD,Hydrogen,sed,season -TPASKM,Passenger kms,svd,daynight -RSHEAT,Residential heating,svd,daynight -CO2EMT,CO2 emitted,oth,annual -BIOPRD,Biomass produced,sed,season -BIOPEL,Biomass pellets,sed,season +id,description,type,time_slice_level,units +OILCRD,Crude oil,sed,season,NOUNIT +GASPRD,Gas produced,sed,season,NOUNIT +GASOLI,Gasoline,sed,season,NOUNIT +DIESEL,Diesel,sed,season,NOUNIT +GASNAT,Natural gas,sed,season,NOUNIT +ELCTRI,Electricity,sed,daynight,NOUNIT +H2YPRD,Hydrogen,sed,season,NOUNIT +TPASKM,Passenger kms,svd,daynight,NOUNIT +RSHEAT,Residential heating,svd,daynight,NOUNIT +CO2EMT,CO2 emitted,oth,annual,NOUNIT +BIOPRD,Biomass produced,sed,season,NOUNIT +BIOPEL,Biomass pellets,sed,season,NOUNIT diff --git a/examples/missing_commodity/commodities.csv b/examples/missing_commodity/commodities.csv index eb5ed39c2..cc6c64212 100644 --- a/examples/missing_commodity/commodities.csv +++ b/examples/missing_commodity/commodities.csv @@ -1,8 +1,8 @@ -id,description,type,time_slice_level -GASPRD,Gas produced,sed,season -GASNAT,Natural gas,sed,season -BIOPRD,Biomass produced,sed,season -BIOPEL,Biomass pellets,sed,season -ELCTRI,Electricity,sed,daynight -RSHEAT,Residential heating,svd,daynight -CO2EMT,CO2 emitted,oth,annual +id,description,type,time_slice_level,units +GASPRD,Gas produced,sed,season,NOUNIT +GASNAT,Natural gas,sed,season,NOUNIT +BIOPRD,Biomass produced,sed,season,NOUNIT +BIOPEL,Biomass pellets,sed,season,NOUNIT +ELCTRI,Electricity,sed,daynight,NOUNIT +RSHEAT,Residential heating,svd,daynight,NOUNIT +CO2EMT,CO2 emitted,oth,annual,NOUNIT diff --git a/examples/muse1_default/commodities.csv b/examples/muse1_default/commodities.csv index c32259118..2595792a6 100644 --- a/examples/muse1_default/commodities.csv +++ b/examples/muse1_default/commodities.csv @@ -1,6 +1,6 @@ -id,description,type,time_slice_level -electricity,Electricity,sed,daynight -gas,Gas,sed,daynight -heat,Heat,svd,daynight -wind,Wind,oth,daynight -CO2f,Carbon dioxide,oth,daynight +id,description,type,time_slice_level,units +electricity,Electricity,sed,daynight,PJ +gas,Gas,sed,daynight,PJ +heat,Heat,svd,daynight,PJ +wind,Wind,oth,daynight,PJ +CO2f,Carbon dioxide,oth,daynight,kt diff --git a/examples/simple/commodities.csv b/examples/simple/commodities.csv index dd5f76af3..1e6bd6b04 100644 --- a/examples/simple/commodities.csv +++ b/examples/simple/commodities.csv @@ -1,6 +1,6 @@ -id,description,type,time_slice_level -GASPRD,Gas produced,sed,season -GASNAT,Natural gas,sed,season -ELCTRI,Electricity,sed,daynight -RSHEAT,Residential heating,svd,daynight -CO2EMT,CO2 emitted,oth,annual +id,description,type,time_slice_level,units +GASPRD,Gas produced,sed,season,NOUNIT +GASNAT,Natural gas,sed,season,NOUNIT +ELCTRI,Electricity,sed,daynight,NOUNIT +RSHEAT,Residential heating,svd,daynight,NOUNIT +CO2EMT,CO2 emitted,oth,annual,NOUNIT diff --git a/examples/two_outputs/commodities.csv b/examples/two_outputs/commodities.csv index e8f247b54..1c8b3819d 100644 --- a/examples/two_outputs/commodities.csv +++ b/examples/two_outputs/commodities.csv @@ -1,12 +1,12 @@ -id,description,type,time_slice_level -OILCRD,Crude oil,sed,season -GASPRD,Gas produced,sed,season -GASOLI,Gasoline,sed,season -DIESEL,Diesel,sed,season -GASNAT,Natural gas,sed,season -ELCTRI,Electricity,sed,daynight -TPASKM,Passenger kms,svd,daynight -RSHEAT,Residential heating,svd,daynight -CO2EMT,CO2 emitted,oth,annual -BIOPRD,Biomass produced,sed,season -BIOPEL,Biomass pellets,sed,season +id,description,type,time_slice_level,units +OILCRD,Crude oil,sed,season,NOUNIT +GASPRD,Gas produced,sed,season,NOUNIT +GASOLI,Gasoline,sed,season,NOUNIT +DIESEL,Diesel,sed,season,NOUNIT +GASNAT,Natural gas,sed,season,NOUNIT +ELCTRI,Electricity,sed,daynight,NOUNIT +TPASKM,Passenger kms,svd,daynight,NOUNIT +RSHEAT,Residential heating,svd,daynight,NOUNIT +CO2EMT,CO2 emitted,oth,annual,NOUNIT +BIOPRD,Biomass produced,sed,season,NOUNIT +BIOPEL,Biomass pellets,sed,season,NOUNIT diff --git a/examples/two_regions/commodities.csv b/examples/two_regions/commodities.csv index c32259118..2595792a6 100644 --- a/examples/two_regions/commodities.csv +++ b/examples/two_regions/commodities.csv @@ -1,6 +1,6 @@ -id,description,type,time_slice_level -electricity,Electricity,sed,daynight -gas,Gas,sed,daynight -heat,Heat,svd,daynight -wind,Wind,oth,daynight -CO2f,Carbon dioxide,oth,daynight +id,description,type,time_slice_level,units +electricity,Electricity,sed,daynight,PJ +gas,Gas,sed,daynight,PJ +heat,Heat,svd,daynight,PJ +wind,Wind,oth,daynight,PJ +CO2f,Carbon dioxide,oth,daynight,kt diff --git a/schemas/input/commodities.yaml b/schemas/input/commodities.yaml index d88ffe993..40a21f2c2 100644 --- a/schemas/input/commodities.yaml +++ b/schemas/input/commodities.yaml @@ -42,3 +42,11 @@ fields: If unspecified, the commodity will use the default pricing strategy according to the commodity type: `shadow` for `svd` and `sed` commodities, and `unpriced` for `oth` commodities. + - name: units + type: string + description: The units of measurement for this commodity + notes: | + This field is used for validation only and is not used for internal unit conversions. MUSE + will validate that all SED/SVD output flows from the same process have the same units, + as the annual fixed costs are distributed proportionally between outputs based on flow coefficients. + E.g. `Petajoules`, `Tonnes`. diff --git a/src/commodity.rs b/src/commodity.rs index ab41779d6..c949e0226 100644 --- a/src/commodity.rs +++ b/src/commodity.rs @@ -54,6 +54,9 @@ pub struct Commodity { /// `time_slice_level` field. E.g. if the `time_slice_level` is seasonal, then there will be /// keys representing each season (and not e.g. individual time slices). pub demand: DemandMap, + /// Units for this commodity represented as a string e.g Petajoules, Tonnes + /// This is only used for validation purposes. + pub units: String, } define_id_getter! {Commodity, CommodityID} diff --git a/src/fixture.rs b/src/fixture.rs index aa43feb6d..819923ce2 100644 --- a/src/fixture.rs +++ b/src/fixture.rs @@ -150,6 +150,7 @@ pub fn svd_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), } } @@ -164,6 +165,7 @@ pub fn sed_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), } } @@ -178,6 +180,37 @@ pub fn other_commodity() -> Commodity { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), + } +} + +#[fixture] +pub fn sed_commodity_pj() -> Commodity { + Commodity { + id: "sed_pj".into(), + description: "Test SED commodity (PJ)".into(), + kind: CommodityType::SupplyEqualsDemand, + time_slice_level: TimeSliceLevel::DayNight, + pricing_strategy: PricingStrategy::Shadow, + levies_prod: CommodityLevyMap::new(), + levies_cons: CommodityLevyMap::new(), + demand: DemandMap::new(), + units: "PJ".into(), + } +} + +#[fixture] +pub fn sed_commodity_tonnes() -> Commodity { + Commodity { + id: "sed_tonnes".into(), + description: "Test SED commodity (tonnes)".into(), + kind: CommodityType::SupplyEqualsDemand, + time_slice_level: TimeSliceLevel::DayNight, + pricing_strategy: PricingStrategy::Shadow, + levies_prod: CommodityLevyMap::new(), + levies_cons: CommodityLevyMap::new(), + demand: DemandMap::new(), + units: "tonnes".into(), } } @@ -415,14 +448,14 @@ mod tests { #[test] fn patch_and_validate_simple_fail() { let patch = FilePatch::new("commodities.csv") - .with_deletion("RSHEAT,Residential heating,svd,daynight"); + .with_deletion("RSHEAT,Residential heating,svd,daynight,NOUNIT"); assert!(patch_and_validate_simple!(vec![patch]).is_err()); } #[test] fn patch_and_run_simple_fail() { let patch = FilePatch::new("commodities.csv") - .with_deletion("RSHEAT,Residential heating,svd,daynight"); + .with_deletion("RSHEAT,Residential heating,svd,daynight,NOUNIT"); assert!(patch_and_run_simple!(vec![patch]).is_err()); } } diff --git a/src/input/agent/commodity_portion.rs b/src/input/agent/commodity_portion.rs index 11054d1dc..4fd971d80 100644 --- a/src/input/agent/commodity_portion.rs +++ b/src/input/agent/commodity_portion.rs @@ -226,6 +226,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }), )]); @@ -269,6 +270,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }), ); assert!( diff --git a/src/input/commodity.rs b/src/input/commodity.rs index 32a232933..2a0bf22f9 100644 --- a/src/input/commodity.rs +++ b/src/input/commodity.rs @@ -30,6 +30,7 @@ struct CommodityRaw { pub kind: CommodityType, pub time_slice_level: TimeSliceLevel, pub pricing_strategy: Option, + pub units: String, } /// Read commodity data from the specified model directory. @@ -119,6 +120,7 @@ where levies_prod: CommodityLevyMap::default(), levies_cons: CommodityLevyMap::default(), demand: DemandMap::default(), + units: commodity_raw.units, }; validate_commodity(&commodity)?; @@ -200,6 +202,7 @@ mod tests { levies_prod: CommodityLevyMap::default(), levies_cons: CommodityLevyMap::default(), demand: DemandMap::default(), + units: "PJ".into(), } } diff --git a/src/input/process/flow.rs b/src/input/process/flow.rs index fc7b769ed..abcf23947 100644 --- a/src/input/process/flow.rs +++ b/src/input/process/flow.rs @@ -1,6 +1,6 @@ //! Code for reading process flows from a CSV file. use super::super::{input_err_msg, read_csv}; -use crate::commodity::{CommodityID, CommodityMap}; +use crate::commodity::{CommodityID, CommodityMap, CommodityType}; use crate::process::{ FlowDirection, FlowType, ProcessFlow, ProcessFlowsMap, ProcessID, ProcessMap, }; @@ -11,7 +11,7 @@ use anyhow::{Context, Result, ensure}; use indexmap::IndexMap; use itertools::iproduct; use serde::Deserialize; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::Path; use std::rc::Rc; @@ -81,6 +81,40 @@ pub fn read_process_flows( .with_context(|| input_err_msg(&file_path)) } +/// Validates that all SED/SVD outputs from each process have consistent units. +/// +/// For processes with multiple SED/SVD outputs, the annual fixed costs are distributed +/// proportionally based on flow coefficients. This only makes sense if all outputs share +/// the same units. +fn validate_output_flows_units(flows_map: &HashMap) -> Result<()> { + // for each map of process flows corresponding to a process + for (process_id, process_flows) in flows_map { + // for each region/year of the process flows + for ((region_id, year), flows) in process_flows { + let sed_svd_output_units: HashSet<&str> = flows + .values() + .filter_map(|flow| { + let commodity = &flow.commodity; + (flow.coeff.value() > 0.0 + && matches!( + commodity.kind, + CommodityType::ServiceDemand | CommodityType::SupplyEqualsDemand + )) + .then_some(commodity.units.as_str()) + }) + .collect(); + + ensure!( + sed_svd_output_units.len() <= 1, + "Process '{process_id}' has SED/SVD outputs with different units \ + (region: {region_id}, year: {year}): {sed_svd_output_units:?}" + ); + } + } + + Ok(()) +} + /// Read `ProcessFlowRaw` records from an iterator and convert them into `ProcessFlow` records. /// /// # Arguments @@ -160,6 +194,7 @@ where validate_flows_and_update_primary_output(processes, &flows_map, milestone_years)?; validate_secondary_flows(processes, &flows_map, milestone_years)?; + validate_output_flows_units(&flows_map)?; Ok(flows_map) } @@ -339,8 +374,9 @@ mod tests { use super::*; use crate::commodity::Commodity; use crate::fixture::{ - assert_error, assert_validate_fails_with_simple, assert_validate_ok_simple, process, - sed_commodity, svd_commodity, + assert_error, assert_validate_fails_with_simple, assert_validate_ok_simple, + other_commodity, process, sed_commodity, sed_commodity_pj, sed_commodity_tonnes, + svd_commodity, }; use crate::patch::FilePatch; use crate::process::{FlowType, Process, ProcessFlow, ProcessMap}; @@ -380,6 +416,104 @@ mod tests { (processes, flows) } + #[rstest] + fn output_flows_matching_units( + svd_commodity: Commodity, + sed_commodity: Commodity, + process: Process, + ) { + // Both commodities have the same units + assert_eq!(svd_commodity.units, sed_commodity.units); + + let commodity1 = Rc::new(svd_commodity); + let commodity2 = Rc::new(sed_commodity); + let (_, flows_map) = build_maps( + process, + [ + (commodity1.id.clone(), flow(commodity1.clone(), 1.0)), + (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), + ] + .into_iter(), + None, + ); + + // Validation should pass since the units are the same + validate_output_flows_units(&flows_map).unwrap(); + } + + #[rstest] + fn output_flows_mismatched_units( + sed_commodity_pj: Commodity, + sed_commodity_tonnes: Commodity, + process: Process, + ) { + // Ensure the two commodities have different units + assert_ne!(sed_commodity_pj.units, sed_commodity_tonnes.units); + + let commodity1 = Rc::new(sed_commodity_pj); + let commodity2 = Rc::new(sed_commodity_tonnes); + let (_, flows_map) = build_maps( + process, + [ + (commodity1.id.clone(), flow(commodity1.clone(), 1.0)), + (commodity2.id.clone(), flow(commodity2.clone(), 2.0)), + ] + .into_iter(), + None, + ); + + // Different units should cause validation to fail + let result = validate_output_flows_units(&flows_map); + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .to_string() + .contains("has SED/SVD outputs with different units") + ); + } + + #[rstest] + fn output_flows_other_commodity_ignored( + sed_commodity_pj: Commodity, + other_commodity: Commodity, + process: Process, + ) { + // Modify OTH commodity to have different units + let mut other_commodity = other_commodity; + other_commodity.units = "tonnes".into(); + assert_ne!(sed_commodity_pj.units, other_commodity.units); + + let sed_commodity = Rc::new(sed_commodity_pj); + let oth_commodity = Rc::new(other_commodity); + + let (_, flows_map) = build_maps( + process, + [ + (sed_commodity.id.clone(), flow(sed_commodity.clone(), 1.0)), + (oth_commodity.id.clone(), flow(oth_commodity.clone(), 2.0)), + ] + .into_iter(), + None, + ); + + // OTH commodity should be ignored, validation should pass + validate_output_flows_units(&flows_map).unwrap(); + } + + #[rstest] + fn single_sed_svd_output(svd_commodity: Commodity, process: Process) { + let commodity = Rc::new(svd_commodity); + let (_, flows_map) = build_maps( + process, + std::iter::once((commodity.id.clone(), flow(commodity.clone(), 1.0))), + None, + ); + + // Single output should always pass validation + validate_output_flows_units(&flows_map).unwrap(); + } + #[rstest] fn single_output_infer_primary(#[from(svd_commodity)] commodity: Commodity, process: Process) { let milestone_years = vec![2010, 2020]; diff --git a/src/process.rs b/src/process.rs index f12986e16..f5bf6fcfc 100644 --- a/src/process.rs +++ b/src/process.rs @@ -576,6 +576,7 @@ mod tests { levies_prod, levies_cons, demand: DemandMap::new(), + units: "PJ".into(), }) } @@ -596,6 +597,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: levies, demand: DemandMap::new(), + units: "PJ".into(), }) } @@ -616,6 +618,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }) } @@ -638,6 +641,7 @@ mod tests { levies_prod, levies_cons, demand: DemandMap::new(), + units: "PJ".into(), }) } @@ -652,6 +656,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }) } @@ -667,6 +672,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }), coeff: FlowPerActivity(1.0), kind: FlowType::Fixed, @@ -689,6 +695,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }), coeff: FlowPerActivity(1.0), kind: FlowType::Fixed, @@ -711,6 +718,7 @@ mod tests { levies_prod: levies, levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }), coeff: FlowPerActivity(1.0), kind: FlowType::Fixed, @@ -978,6 +986,7 @@ mod tests { levies_prod: CommodityLevyMap::new(), levies_cons: CommodityLevyMap::new(), demand: DemandMap::new(), + units: "PJ".into(), }); let flow_in = ProcessFlow { diff --git a/tests/add_column.sh b/tests/add_column.sh new file mode 100755 index 000000000..add372150 --- /dev/null +++ b/tests/add_column.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# add_column.sh +# Adds a new column to a specified CSV file across all example subdirectories +# Usage: add_column.sh + +set -e + +# Validate command line arguments +if [ $# -ne 3 ]; then + echo "Usage: $0 " >&2 + echo "Example: $0 commodities.csv units NOUNIT" >&2 + exit 1 +fi + +csv_filename=$1 +column_name=$2 +default_value=$3 + +# Determine the examples directory relative to the tests directory +script_dir=$(dirname "$0") +examples_dir="$script_dir/../examples" + +# Verify the examples directory exists +if [ ! -d "$examples_dir" ]; then + echo "Error: Examples directory not found at $examples_dir" >&2 + exit 1 +fi + +# Process each subdirectory in the examples folder +for subdir in "$examples_dir"/*; do + # Skip if not a directory + [ -d "$subdir" ] || continue + + csv_path="$subdir/$csv_filename" + + # Skip if the CSV file doesn't exist in this subdirectory + if [ ! -f "$csv_path" ]; then + continue + fi + + echo "Processing: $csv_path" + + # Create a temporary file for the modified CSV + temp_file=$(mktemp) + + # Read and process the CSV file + first_line=true + while IFS= read -r line || [ -n "$line" ]; do + if $first_line; then + # Append new column name to header + echo "$line,$column_name" >> "$temp_file" + first_line=false + else + # Append default value to each data row + echo "$line,$default_value" >> "$temp_file" + fi + done < "$csv_path" + + # Replace original file with modified version + mv "$temp_file" "$csv_path" +done + +echo "Column addition complete"