List of functionalities shared by sensors

[matthew-schultz]

Bullet list of functionalities a new sensor would like to have, but not necessarily needed to get data:

• error_log id
• database
• verbose flag
• txt error.log file

Add to wiki:

• DB Schema for each bullet
  • fade the tables that do not get affected
  • highlight those that do
    Should be useful for future users of lonoa to understand what db columns the scripts do and do not use.

[matthew-schultz]

@carlosparadis just wanted to annotate the shared current parts of the code in the master branch that we talked about before I head out for Christmas vacation

error_logging

API_EGAUGE
get_data_from_api()

lonoa/egauge/script/api_egauge.py

Lines 141 to 144 in 65609c5

	for purpose_sensor in purpose_sensors:
	error_log_row = orm_egauge.ErrorLog(purpose_id=purpose_sensor.purpose_id, datetime=current_time, was_success=True, pipeline_stage=orm_egauge.ErrorLog.PipelineStageEnum.data_acquisition)
	conn.add(error_log_row)
	conn.commit()

insert_readings_into_database()

lonoa/egauge/script/api_egauge.py

Lines 193 to 204 in 65609c5

	error_log_row = orm_egauge.ErrorLog(purpose_id=purpose_sensor.purpose_id, datetime=current_time, pipeline_stage=orm_egauge.ErrorLog.PipelineStageEnum.database_insertion, was_success=True)
	conn.add(error_log_row)
	# need to flush and refresh to get error_log_row.log_id
	conn.flush()
	conn.refresh(error_log_row)
	# update current set of readings with related log_id
	conn.query(orm_egauge.Reading.log_id).\
	filter(orm_egauge.Reading.purpose_id == purpose_sensor.purpose_id,
	orm_egauge.Reading.upload_timestamp == current_time).\
	update({'log_id':error_log_row.log_id})
	logging.info(str(rows_inserted) + ' readings(s) attempted to be inserted by ' + SCRIPT_NAME + ' for purpose id ' + str(purpose_sensor.purpose_id))
	conn.commit()

log_failure_to_connect_to_api()

lonoa/egauge/script/api_egauge.py

Lines 214 to 217 in 65609c5

	for purpose_id in purpose_ids:
	error_log_row = orm_egauge.ErrorLog(datetime=current_time, error_type=exception.__class__.__name__, pipeline_stage=orm_egauge.ErrorLog.PipelineStageEnum.data_acquisition, purpose_id=purpose_id, was_success=False)
	conn.add(error_log_row)
	conn.commit()

log_failure_to_connect_to_database()

lonoa/egauge/script/api_egauge.py

Lines 227 to 230 in 65609c5

	for purpose_sensor in purpose_sensors:
	error_log_row = orm_egauge.ErrorLog(datetime=current_time, error_type=exception.__class__.__name__, purpose_id=purpose_sensor.purpose_id, pipeline_stage=orm_egauge.ErrorLog.PipelineStageEnum.database_insertion, was_success=False)
	conn.add(error_log_row)
	conn.commit()

---

API_WEBCTRL
get_data_from_api()

lonoa/webctrl/script/api_webctrl.py

Lines 113 to 117 in 65609c5

	conn.add(error_log_row)
	conn.commit()
	return readings
	else:
	readings.raise_for_status()

insert_readings_into_database()

lonoa/webctrl/script/api_webctrl.py

Lines 172 to 182 in 65609c5

	error_log_row = orm_webctrl.ErrorLog(datetime=current_time, was_success=True, purpose_id=sensor.purpose_id, pipeline_stage=orm_webctrl.ErrorLog.PipelineStageEnum.database_insertion)
	conn.add(error_log_row)
	# need to flush and refresh to get error_log_row.log_id
	conn.flush()
	conn.refresh(error_log_row)
	# update current set of readings with related log_id
	conn.query(orm_webctrl.Reading.log_id). \
	filter(orm_webctrl.Reading.purpose_id == sensor.purpose_id,
	orm_webctrl.Reading.upload_timestamp == current_time). \
	update({'log_id': error_log_row.log_id})
	conn.commit()

log_failure_to_connect_to_api()

lonoa/webctrl/script/api_webctrl.py

Lines 191 to 193 in 65609c5

	error_log_row = orm_webctrl.ErrorLog(datetime=current_time, error_type=exception.__class__.__name__, pipeline_stage=orm_webctrl.ErrorLog.PipelineStageEnum.data_acquisition, purpose_id=sensor.purpose_id, was_success=False)
	conn.add(error_log_row)
	conn.commit()

log_failure_to_connect_to_database()

lonoa/webctrl/script/api_webctrl.py

Lines 203 to 205 in 65609c5

	error_log_row = orm_webctrl.ErrorLog(datetime=current_time, error_type=exception.__class__.__name__, pipeline_stage=orm_webctrl.ErrorLog.PipelineStageEnum.database_insertion, purpose_id=sensor.purpose_id, was_success=False)
	conn.add(error_log_row)
	conn.commit()

---

EXTRACT_HOBO
get_csv_from_folder_not_in_db()

lonoa/hobo/script/extract_hobo.py

Lines 134 to 136 in 65609c5

	for sensor_info_row in sensor_info_rows:
	error_log_row = orm_hobo.ErrorLog(was_success=True, purpose_id=sensor_info_row.purpose_id, datetime=current_time, pipeline_stage=orm_hobo.ErrorLog.PipelineStageEnum.data_acquisition)
	conn.add(error_log_row)

insert_csv_readings_into_db()

lonoa/hobo/script/extract_hobo.py

Lines 194 to 195 in 65609c5

	error_log_row = orm_hobo.ErrorLog(was_success=True, purpose_id=sensor_info_row.purpose_id, datetime=current_time, pipeline_stage=orm_hobo.ErrorLog.PipelineStageEnum.database_insertion)
	conn.add(error_log_row)

log_failure_to_get_csv_readings_from_folder_not_in_db()

lonoa/hobo/script/extract_hobo.py

Lines 219 to 226 in 65609c5

	error_log_row = orm_hobo.ErrorLog(was_success=False, datetime=current_time, error_type=exception.__class__.__name__, pipeline_stage=orm_hobo.ErrorLog.PipelineStageEnum.data_acquisition)
	conn.add(error_log_row)
	# need to flush and refresh to get error_log_row.log_id
	conn.flush()
	conn.refresh(error_log_row)
	csv_filename_row = orm_hobo.ErrorLogDetails(log_id=error_log_row.log_id, information_type="csv_filename", information_value=csv_filename)
	conn.add(csv_filename_row)
	conn.commit()

log_failure_to_insert_csv_readings_into_db()

lonoa/hobo/script/extract_hobo.py

Lines 238 to 243 in 65609c5

	if not new_readings:
	error_log_row = orm_hobo.ErrorLog(purpose_id=sensor_info_row.purpose_id, datetime=current_time, error_type=exception.__class__.__name__,pipeline_stage=orm_hobo.ErrorLog.PipelineStageEnum.database_insertion)
	# set was_success to False if readings were new but an error was thrown
	else:
	error_log_row = orm_hobo.ErrorLog(purpose_id=sensor_info_row.purpose_id, was_success=False, datetime=current_time, error_type=exception.__class__.__name__, pipeline_stage=orm_hobo.ErrorLog.PipelineStageEnum.database_insertion)
	conn.add(error_log_row)