From bc27c710413f491527d6b1f83dabb62d41589b98 Mon Sep 17 00:00:00 2001 From: junos Date: Thu, 24 Dec 2020 16:46:55 +0100 Subject: [PATCH] Copy Models and edit for imports. --- config/ORMtutorial.ipynb | 175 ++++++++++++++++++++++ config/environment.yml | 1 + config/models.py | 315 +++++++++++++++++++++++++++++++++++++++ setup.py | 6 +- 4 files changed, 496 insertions(+), 1 deletion(-) create mode 100644 config/ORMtutorial.ipynb create mode 100644 config/models.py diff --git a/config/ORMtutorial.ipynb b/config/ORMtutorial.ipynb new file mode 100644 index 0000000..7c6bd97 --- /dev/null +++ b/config/ORMtutorial.ipynb @@ -0,0 +1,175 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'1.3.21'" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import sqlalchemy\n", + "sqlalchemy.__version__ " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import create_engine\n", + "engine = create_engine('sqlite:///:memory:', echo=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.ext.declarative import declarative_base\n", + "Base = declarative_base()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import Column, Integer, String\n", + "class User(Base):\n", + " __tablename__ = 'users'\n", + "\n", + " id = Column(Integer, primary_key=True)\n", + " name = Column(String)\n", + " fullname = Column(String)\n", + " nickname = Column(String)\n", + "\n", + " def __repr__(self):\n", + " return \"\" % (\n", + " self.name, self.fullname, self.nickname)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Table('users', MetaData(bind=None), Column('id', Integer(), table=, primary_key=True, nullable=False), Column('name', String(), table=), Column('fullname', String(), table=), Column('nickname', String(), table=), schema=None)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "User.__table__ " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2020-12-24 16:40:37,230 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1\n", + "2020-12-24 16:40:37,236 INFO sqlalchemy.engine.base.Engine ()\n", + "2020-12-24 16:40:37,239 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1\n", + "2020-12-24 16:40:37,240 INFO sqlalchemy.engine.base.Engine ()\n", + "2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine PRAGMA main.table_info(\"users\")\n", + "2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine ()\n", + "2020-12-24 16:40:37,244 INFO sqlalchemy.engine.base.Engine PRAGMA temp.table_info(\"users\")\n", + "2020-12-24 16:40:37,245 INFO sqlalchemy.engine.base.Engine ()\n", + "2020-12-24 16:40:37,246 INFO sqlalchemy.engine.base.Engine \n", + "CREATE TABLE users (\n", + "\tid INTEGER NOT NULL, \n", + "\tname VARCHAR, \n", + "\tfullname VARCHAR, \n", + "\tnickname VARCHAR, \n", + "\tPRIMARY KEY (id)\n", + ")\n", + "\n", + "\n", + "2020-12-24 16:40:37,247 INFO sqlalchemy.engine.base.Engine ()\n", + "2020-12-24 16:40:37,248 INFO sqlalchemy.engine.base.Engine COMMIT\n" + ] + } + ], + "source": [ + "Base.metadata.create_all(engine)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ed\n", + "edsnickname\n", + "None\n" + ] + } + ], + "source": [ + "ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname')\n", + "print(ed_user.name)\n", + "print(ed_user.nickname)\n", + "print(str(ed_user.id))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy.orm import sessionmaker\n", + "Session = sessionmaker(bind=engine)\n", + "session = Session()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/config/environment.yml b/config/environment.yml index 5447624..55ffcc8 100644 --- a/config/environment.yml +++ b/config/environment.yml @@ -7,6 +7,7 @@ dependencies: - black - isort - flake8 + - jupyterlab - mypy - psycopg2 - python-dotenv diff --git a/config/models.py b/config/models.py new file mode 100644 index 0000000..bdcd81d --- /dev/null +++ b/config/models.py @@ -0,0 +1,315 @@ +from datetime import datetime +from sqlalchemy import Column, String, Boolean, \ + Integer, SmallInteger, BigInteger, \ + Float, TIMESTAMP, \ + ForeignKey, UniqueConstraint +from sqlalchemy.ext.declarative import declarative_base, declared_attr +from sqlalchemy.dialects.postgresql import ARRAY as PSQL_ARRAY, \ + JSONB as PSQL_JSONB, TEXT as PSQL_TEXT, INTEGER as PSQL_INTEGER +from sqlalchemy.orm import relationship + +Base = declarative_base() + + +class Participant(Base): + __tablename__ = "participants" + id = Column(Integer, primary_key=True) + username = Column(String(64), index=True, unique=True) + password_hash = Column(String(128)) + token = Column(String(32), index=True, unique=True) + token_expiration_utc = Column(TIMESTAMP(timezone=False)) + + collection_start_utc = Column(TIMESTAMP(timezone=False)) + last_upload_utc = Column(TIMESTAMP(timezone=False)) + day_count_uploaded = Column(SmallInteger) + last_known_device_id = Column(String(length=36), nullable=False, default="") + + active = Column(Boolean, index=True, nullable=False, default=False) + marked_active_utc = Column(TIMESTAMP(timezone=False)) + day_count_active = Column(SmallInteger) + + tester = Column(Boolean, index=True, nullable=False, default=False) + + def __repr__(self): + return 'Participant()' + + def __str__(self): + return ''.format(self.username) + + +class AWAREsensor(Base): + id = Column(BigInteger, primary_key=True, nullable=False) + _id = Column(BigInteger, nullable=False) + timestamp = Column(BigInteger, nullable=False) + device_id = Column(String(length=36), nullable=False) + + @declared_attr + def __tablename__(cls): + return cls.__name__.lower() + + @declared_attr + def participant_id(cls): + return Column(Integer, ForeignKey('participants.id'), nullable=False, index=True) + + @declared_attr + def participants(cls): + return relationship('Participant', lazy="select", backref=cls.__tablename__) + + @declared_attr + def __table_args__(cls): + return UniqueConstraint('device_id', '_id', name=cls.__tablename__ + "_twice"), + # I think it makes more sense to create a Constraint on device_id and _id rather than relate it to participant_id. + # _id is a primary key, auto incremented by AWARE. + # However, I would expect it to reset back to 1 if the application was reinstalled, + # similarly to how device_id resets. + + +class Accelerometer(Base, AWAREsensor): + double_values_0 = Column(Float, nullable=False) + double_values_1 = Column(Float, nullable=False) + double_values_2 = Column(Float, nullable=False) + accuracy = Column(SmallInteger, nullable=True) + label = Column(String, nullable=True) + + +class GoogleAR(Base, AWAREsensor): + __tablename__ = "google_ar" + activity_name = Column(String(32), nullable=True) + activity_type = Column(SmallInteger, nullable=True) + confidence = Column(SmallInteger, nullable=True) + activities = Column(PSQL_JSONB(none_as_null=False), nullable=True) + + +class Application(Base, AWAREsensor): + __tablename__ = "applications" +# package_name = Column(String, nullable=False) +# application_name = Column(String) + package_hash = Column(String(64), nullable=False) + play_store_genre = Column(String, nullable=True) + is_system_app = Column(Boolean) + + +class Barometer(Base, AWAREsensor): + double_values_0 = Column(Float, nullable=False) + accuracy = Column(SmallInteger, nullable=True) + label = Column(String, nullable=True) + + +class BarometerSensor(Base, AWAREsensor): + __tablename__ = "barometer_sensor" + # Since this table is not really important, I will leave all columns as nullable. (nullable=True by default.) + double_sensor_maximum_range = Column(Float) + double_sensor_minimum_delay = Column(Float) + sensor_name = Column(String) + double_sensor_power_ma = Column(Float) + double_sensor_resolution = Column(Float) + sensor_type = Column(String) + sensor_vendor = Column(String) + sensor_version = Column(String) + + +class Battery(Base, AWAREsensor): + battery_status = Column(SmallInteger, nullable=False) + battery_level = Column(SmallInteger, nullable=False) + battery_scale = Column(SmallInteger, nullable=False) + battery_voltage = Column(Integer, nullable=True) + battery_temperature = Column(Integer, nullable=True) + battery_adaptor = Column(SmallInteger, nullable=True) + battery_health = Column(SmallInteger, nullable=True) + battery_technology = Column(String, nullable=True) + + +class Bluetooth(Base, AWAREsensor): + bt_address = Column(String(length=40), nullable=False) + bt_name = Column(String, nullable=True) + bt_rssi = Column(Integer, nullable=True) + label = Column(String, nullable=True) + + +class Call(Base, AWAREsensor): + call_type = Column(SmallInteger, nullable=False) + call_duration = Column(Integer, nullable=False) + trace = Column(String(length=40), nullable=True) + + +class ESM(Base, AWAREsensor): + esm_status = Column(SmallInteger, nullable=False) + esm_user_answer = Column(String, nullable=True) + esm_notification_timeout = Column(SmallInteger) + esm_json = Column(PSQL_JSONB(none_as_null=False), nullable=False) + double_esm_user_answer_timestamp = Column(BigInteger, nullable=False) + esm_trigger = Column(String, nullable=True) + esm_session = Column(Integer, nullable=False) + esm_notification_id = Column(Integer, nullable=False) + esm_expiration_threshold = Column(SmallInteger) + + +class Imperfection(Base): + id = Column(BigInteger, primary_key=True, nullable=False) + timestamp = Column(BigInteger, nullable=False) + error = Column(String) + data_type = Column(String) + data = Column(String) + + +class LightSensor(Base, AWAREsensor): + __tablename__ = "light_sensor" + double_light_lux = Column(Float, nullable=False) + accuracy = Column(Integer, nullable=True) + label = Column(String, nullable=True) + + +class Location(Base, AWAREsensor): + __tablename__ = "locations" + double_latitude = Column(Float, nullable=False) + double_longitude = Column(Float, nullable=False) + double_bearing = Column(Float) + double_speed = Column(Float) + double_altitude = Column(Float) + provider = Column(String) + accuracy = Column(Integer) + label = Column(String, nullable=True) + category = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True) + category_short = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True) + distance = Column(PSQL_ARRAY(PSQL_INTEGER), nullable=True) + api_response_code = Column(SmallInteger, nullable=False) + api_response = Column(String, nullable=True) + + +class Mic(Base, AWAREsensor): + half_s_speech = Column(Boolean, nullable=False) + rnnoutp = Column(Float, nullable=False) + flatness = Column(Float, nullable=False) + absmax = Column(Float, nullable=False) + max = Column(Float, nullable=False) + min = Column(Float, nullable=False) + # mfcc = Column(PSQL_ARRAY(PSQL_REAL, dimensions=13), nullable=False) + mfcc_0 = Column(Float, nullable=False) + mfcc_1 = Column(Float, nullable=False) + mfcc_2 = Column(Float, nullable=False) + mfcc_3 = Column(Float, nullable=False) + mfcc_4 = Column(Float, nullable=False) + mfcc_5 = Column(Float, nullable=False) + mfcc_6 = Column(Float, nullable=False) + mfcc_7 = Column(Float, nullable=False) + mfcc_8 = Column(Float, nullable=False) + mfcc_9 = Column(Float, nullable=False) + mfcc_10 = Column(Float, nullable=False) + mfcc_11 = Column(Float, nullable=False) + mfcc_12 = Column(Float, nullable=False) + + +class Speech(Base, AWAREsensor): + speech_proportion = Column(Float, nullable=False) + + +class NetworkData(Base, AWAREsensor): + __tablename__ = "network_data" + network_type = Column(SmallInteger, nullable=False) + network_subtype = Column(String(10), nullable=False) + network_state = Column(SmallInteger, nullable=True) + + +class NetworkTraffic(Base, AWAREsensor): + __tablename__ = "network_traffic_data" + network_type = Column(SmallInteger, nullable=False) + double_received_bytes = Column(Float, nullable=True) + double_sent_bytes = Column(Float, nullable=True) + double_received_packets = Column(Float, nullable=True) + double_sent_packets = Column(Float, nullable=True) + + +class Notification(Base, AWAREsensor): + __tablename__ = "notifications" +# package_name = Column(String, nullable=False) +# application_name = Column(String, nullable=False) + package_hash = Column(String(64), nullable=False) + play_store_genre = Column(String, nullable=True) + sound = Column(String, nullable=False) + vibrate = Column(String, nullable=False) + sound_app = Column(String, nullable=False) + vibrate_app = Column(String, nullable=False) + led_app = Column(String, nullable=False) + category = Column(String, nullable=False) + is_ongoing = Column(SmallInteger, nullable=False) + is_clearable = Column(SmallInteger, nullable=False) + is_group = Column(SmallInteger, nullable=True) + + +class Processor(Base, AWAREsensor): + double_last_user = Column(Float, nullable=False) + double_last_system = Column(Float, nullable=False) + double_last_idle = Column(Float, nullable=False) + double_user_load = Column(Float, nullable=False) + double_system_load = Column(Float, nullable=False) + double_idle_load = Column(Float, nullable=False) + + +class Proximity(Base, AWAREsensor): + double_proximity = Column(Float, nullable=False) + accuracy = Column(SmallInteger, nullable=True) + label = Column(String, nullable=True) + + +class Screen(Base, AWAREsensor): + screen_status = Column(SmallInteger) + + +class SMS(Base, AWAREsensor): + message_type = Column(SmallInteger, nullable=False) + trace = Column(String(length=40), nullable=False) + + +class Temperature(Base, AWAREsensor): + temperature_celsius = Column(Float, nullable=False) + accuracy = Column(SmallInteger, nullable=True) + label = Column(String, nullable=True) + + +class TemperatureSensor(Base, AWAREsensor): + # I left all of these nullable, as we haven't seen any data from this sensor anyway. + __tablename__ = "temperature_sensor" + double_sensor_maximum_range = Column(Float) + double_sensor_minimum_delay = Column(Float) + sensor_name = Column(String) + double_sensor_power_ma = Column(Float) + double_sensor_resolution = Column(Float) + sensor_type = Column(String) + sensor_vendor = Column(String) + sensor_version = Column(String) + + +class Timezone(Base, AWAREsensor): + timezone = Column(String, nullable=False) + + +class WiFi(Base, AWAREsensor): + bssid = Column(String(length=40), nullable=False) + ssid = Column(String, nullable=True) + rssi = Column(Integer, nullable=False) + security = Column(String, nullable=True) + frequency = Column(Integer, nullable=True) + label = Column(String, nullable=True) + + +all_AWARE_tables = [ESM, Location, Screen, LightSensor, Call, SMS, Application, Notification, Battery, WiFi, Proximity, + Timezone, Processor, NetworkData, NetworkTraffic, Barometer, BarometerSensor, Temperature, + TemperatureSensor, Bluetooth, Accelerometer, GoogleAR, Speech] + +all_AWARE_table_names = [table.__tablename__ for table in all_AWARE_tables] + + +def increment_one(ii): + return ii + 1 + + +class AppCategories(Base): + __tablename__ = "app_categories" + id = Column(Integer, primary_key=True) + application_name = Column(String, nullable=True) + package_name = Column(String, nullable=False) + package_hash = Column(String(64), index=True, nullable=False, unique=True) + play_store_genre = Column(String, nullable=True) + play_store_response = Column(SmallInteger, nullable=False) + number_of_attempts = Column(SmallInteger, nullable=False, default=0, onupdate=increment_one) + last_attempt = Column(TIMESTAMP(timezone=False), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow) diff --git a/setup.py b/setup.py index 17e8642..b939deb 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ import os import sqlalchemy.engine.url - +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker from dotenv import load_dotenv load_dotenv() @@ -15,3 +16,6 @@ db_uri = sqlalchemy.engine.url.URL( port=5432, database="staw" ) + +db_engine = create_engine('sqlite:///:memory:', echo=True) +Session = sessionmaker(bind=db_engine)