Coverage for test/test_config.py: 97%
197 statements
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
« prev ^ index » next coverage.py v7.2.3, created at 2023-05-04 13:14 +0700
1import gzip, json
2import urllib.request
3from pathlib import Path
4import pytest
5from ..agent_model.Model import Model
6from ..agent_model.util import load_data_file
8config_mapping = {
9 '1h': 'simoc-simdata-1-human-preset.json.gz',
10 '1hrad': 'simoc-simdata-1-human-radish-preset.json.gz',
11 '4h': 'simoc-simdata-4-human-preset.json.gz',
12 '4hg': 'simoc-simdata-4-human-garden-preset.json.gz',
13 '1hg_sam': 'simoc-simdata-sam-1-human-garden-preset.json.gz',
14 'b2_mission1a': 'simoc-simdata-b2-mission-1a.json.gz',
15 'b2_mission1b': 'simoc-simdata-b2-mission-1b.json.gz',
16 'b2_mission2': 'simoc-simdata-b2-mission-2.json.gz',
17}
19# ---------------------------------
20# Download Simdata from simoc.space
21# ---------------------------------
23@pytest.fixture(scope="module")
24def download_simdata():
25 directory = Path.cwd() / 'test' / 'v1_simdata'
27 if not directory.exists():
28 directory.mkdir(parents=True)
30 url = "https://simoc.space/download/simdata/"
31 try:
32 for simdata_name in config_mapping.values():
33 file_path = directory / simdata_name
34 if not file_path.exists():
35 urllib.request.urlretrieve(url + simdata_name, file_path)
36 return True
37 except:
38 return False
40def test_download_simdata(download_simdata):
41 assert download_simdata
43# -------------------------------------------
44# Helper funcs to produce comparision reports
45# -------------------------------------------
47def load_simdata(stem):
48 """Load gzipped simdata file."""
49 fname = config_mapping[stem]
50 with gzip.open(f'test/v1_simdata/{fname}', 'rb') as f:
51 data = json.load(f)
52 return data
54def lpe(predictions, targets):
55 """Lifetime percentage error"""
56 _p = abs(sum(predictions))
57 _t = abs(sum(targets))
58 return 0 if _t == 0 else (_p-_t)/_t * 100
60def compare_records(records, stem):
61 """Generate a report of the differences between data generated by the
62 current model and current active simdata."""
64 # Load current simdata
65 simdata = load_simdata(stem)
66 assert records['step_num'][-1] == simdata['steps'], 'Different step numbers'
67 references = simdata['data']
69 # Generate comparison report
70 reports = {}
71 for agent_id, record in records['agents'].items():
72 substitute_names = {'human': 'human_agent'}
73 old_name = substitute_names.get(agent_id, agent_id)
74 reference = references[old_name]
75 report = {}
77 # Compare storage
78 if record['storage']:
79 report['storage'] = {}
80 for currency, predictions in record['storage'].items():
81 targets = reference['storage'][currency]
82 report['storage'][currency] = lpe(predictions, targets)
84 # Compare flows
85 if record['flows']:
86 if agent_id == 'atmosphere_equalizer':
87 continue # Not part of old records
88 report['flows'] = {}
89 for direction, flows in record['flows'].items():
90 for currency, currency_flows in flows.items():
91 if currency == 'par':
92 continue # Not part of old records
93 per_connection = []
94 for connection, predictions in currency_flows.items():
95 targets = reference['flows'][direction][currency][connection]
96 per_connection.append(lpe(predictions, targets))
97 mean_mape = sum(per_connection) / len(per_connection)
98 report['flows'][f'{direction}_{currency}'] = mean_mape
100 # Compare attributes
101 if record['attributes']:
102 report['attributes'] = {}
103 for attribute, predictions in record['attributes'].items():
104 if 'growth' in reference and attribute in reference['growth']:
105 targets = reference['growth'][attribute]
106 elif 'deprive' in attribute:
107 deprive_attr = attribute[:-8] # e.g. 'co2' from 'co2_deprive'
108 if deprive_attr in reference['deprive']:
109 targets = reference['deprive'][deprive_attr]
110 else:
111 continue
112 elif 'criteria' in attribute:
113 if 'buffer' not in reference:
114 continue
115 criteria_attr = '_'.join(attribute.split('_')[:2])
116 if criteria_attr in reference['buffer']:
117 continue # Broken?
118 targets = reference['buffer'][criteria_attr]
119 elif (attribute in {'age', 'delay_start', 'par_rate', 'photoperiod'} or
120 'growth_factor' in attribute):
121 continue
122 else:
123 assert False, f'Unaccounted for attribute: {attribute}'
124 report['attributes'][attribute] = lpe(predictions, targets)
126 reports[agent_id] = report
127 return reports
130class TestConfigs:
131 def test_config_1h(self):
132 config = load_data_file('config_1h.json')
133 model = Model.from_config(**config, record_initial_state=False)
134 model.run()
135 assert model.elapsed_time.days == 10
136 human = model.agents['human']
137 assert human.active == 1
139 records = model.get_records()
140 comparison_report = compare_records(records, '1h')
141 with open('test/v1_simdata/comparison_report_1h.json', 'w') as f:
142 json.dump(comparison_report, f, indent=2)
143 for agent, report in comparison_report.items():
144 for section, fields in report.items():
145 for field, value in fields.items():
146 exceptions = {
147 # ECLSS components are more active because many
148 # previously had duplicated criteria. e.g.
149 ('crew_habitat_small', 'storage', 'co2'): 3.3,
150 ('crew_habitat_small', 'storage', 'h2o'): 1.3,
151 ('water_storage', 'storage', 'treated'): 5.7,
152 }
153 if (agent, section, field) in exceptions:
154 percent_error = exceptions[(agent, section, field)]
155 else:
156 percent_error = 1
157 # Less than 1% error in lifetime
158 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}'
160 def test_config_1hrad(self):
161 stem = '1hrad'
162 config = load_data_file(f'config_{stem}.json')
163 model = Model.from_config(**config)
164 model.run()
165 human = model.agents['human']
166 assert human.active == 1
168 records = model.get_records()
169 comparison_report = compare_records(records, stem)
170 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
171 json.dump(comparison_report, f, indent=2)
172 for agent, report in comparison_report.items():
173 for section, fields in report.items():
174 for field, value in fields.items():
175 exceptions = {
176 ('solar_pv_array_mars', 'flows', 'out_kwh'): 3.4,
177 ('crew_habitat_small', 'storage', 'co2'): 12.2,
178 ('crew_habitat_small', 'storage', 'h2o'): 3.0,
179 ('greenhouse_small', 'storage', 'co2'): 8.9,
180 ('greenhouse_small', 'storage', 'h2o'): 1.9,
181 # These are mistakenly logged twice in the simdata
182 ('radish', 'flows', 'out_radish'): 50,
183 ('radish', 'flows', 'out_inedible_biomass'): 50,
184 # THis means something different now
185 ('radish', 'attributes', 'growth_rate'): 102,
186 # These were formerly multiplied by amount
187 ('radish', 'attributes', 'in_co2_deprive'): 98,
188 ('radish', 'attributes', 'in_potable_deprive'): 98,
189 ('radish', 'attributes', 'in_fertilizer_deprive'): 98,
190 ('water_storage', 'storage', 'treated'): 7.4,
191 ('nutrient_storage', 'storage', 'inedible_biomass'): 1.5,
192 ('power_storage', 'storage', 'kwh'): 11.1,
193 ('co2_removal_SAWD', 'flows', 'in_co2'): 1.8,
194 ('co2_removal_SAWD', 'flows', 'in_kwh'): 3.5,
195 ('co2_removal_SAWD', 'flows', 'out_co2'): 3.5,
196 ('co2_storage', 'storage', 'co2'): 1.8,
197 }
198 if (agent, section, field) in exceptions:
199 percent_error = exceptions[(agent, section, field)]
200 else:
201 percent_error = 1
202 # Less than 1% error in lifetime
203 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}'
206 def test_config_4h(self):
207 stem = '4h'
208 config = load_data_file(f'config_{stem}.json')
209 model = Model.from_config(**config)
210 model.run()
211 human = model.agents['human']
212 assert human.active == 4
214 records = model.get_records()
215 comparison_report = compare_records(records, stem)
216 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
217 json.dump(comparison_report, f, indent=2)
218 for agent, report in comparison_report.items():
219 for section, fields in report.items():
220 for field, value in fields.items():
221 exceptions = {
222 ('human', 'attributes', 'in_potable_deprive'): 75,
223 ('human', 'attributes', 'in_food_deprive'): 75,
224 ('solar_pv_array_mars', 'flows', 'out_kwh'): 3.3,
225 ('crew_habitat_small', 'storage', 'co2'): 4.9,
226 ('crew_habitat_small', 'storage', 'ch4'): 4.2,
227 ('crew_habitat_small', 'storage', 'h2'): 1.8,
228 ('crew_habitat_small', 'storage', 'h2o'): 4.3,
229 ('water_storage', 'storage', 'urine'): 1.2,
230 ('water_storage', 'storage', 'treated'): 10.3,
231 ('power_storage', 'storage', 'kwh'): 1.6,
232 ('co2_reduction_sabatier', 'flows', 'in_h2'): 1.1,
233 ('co2_reduction_sabatier', 'flows', 'in_co2'): 3.3,
234 ('co2_reduction_sabatier', 'flows', 'in_kwh'): 3.3,
235 ('co2_reduction_sabatier', 'flows', 'out_ch4'): 3.3,
236 ('co2_reduction_sabatier', 'flows', 'out_feces'): 3.3,
237 }
238 if (agent, section, field) in exceptions:
239 percent_error = exceptions[(agent, section, field)]
240 else:
241 percent_error = 1
242 # Less than 1% error in lifetime
243 assert abs(value) < percent_error, f'{agent} {section} {field} error: {value}'
245 def test_config_4hg(self):
246 stem = '4hg'
247 config = load_data_file(f'config_{stem}.json')
248 model = Model.from_config(**config)
249 # For some reason, the simdata only has 2300 steps instead of 2400
250 for i in range(2300):
251 model.step()
252 human = model.agents['human']
253 assert human.active == 4
255 records = model.get_records()
256 comparison_report = compare_records(records, stem)
257 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
258 json.dump(comparison_report, f, indent=2)
261 def test_config_1hg_sam(self):
262 stem = '1hg_sam'
263 config = load_data_file(f'config_{stem}.json')
264 model = Model.from_config(**config)
265 model.run()
266 human = model.agents['human']
267 assert human.active == 1
269 records = model.get_records()
270 comparison_report = compare_records(records, stem)
271 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
272 json.dump(comparison_report, f, indent=2)
275 def test_config_b2_mission1a(self):
276 stem = 'b2_mission1a'
277 config = load_data_file(f'config_{stem}.json')
278 model = Model.from_config(**config)
279 model.run()
280 human = model.agents['human']
281 assert human.active == 8
283 records = model.get_records()
284 comparison_report = compare_records(records, stem)
285 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
286 json.dump(comparison_report, f, indent=2)
289 def test_config_b2_mission1b(self):
290 stem = 'b2_mission1b'
291 config = load_data_file(f'config_{stem}.json')
292 model = Model.from_config(**config)
293 model.run()
294 human = model.agents['human']
295 assert human.active == 8
297 records = model.get_records()
298 comparison_report = compare_records(records, stem)
299 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
300 json.dump(comparison_report, f, indent=2)
304 def test_config_b2_mission2(self):
305 stem = 'b2_mission2'
306 config = load_data_file(f'config_{stem}.json')
307 model = Model.from_config(**config)
308 model.run()
309 human = model.agents['human']
310 assert human.active == 8
312 records = model.get_records()
313 comparison_report = compare_records(records, stem)
314 with open(f'test/v1_simdata/comparison_report_{stem}.json', 'w') as f:
315 json.dump(comparison_report, f, indent=2)