Copy Models and edit for imports.

config/ORMtutorial.ipynb

@@ -0,0 +1,175 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'1.3.21'"
+      ]
+     },
+     "execution_count": 1,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import sqlalchemy\n",
+    "sqlalchemy.__version__ "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy import create_engine\n",
+    "engine = create_engine('sqlite:///:memory:', echo=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy.ext.declarative import declarative_base\n",
+    "Base = declarative_base()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy import Column, Integer, String\n",
+    "class User(Base):\n",
+    "    __tablename__ = 'users'\n",
+    "\n",
+    "    id = Column(Integer, primary_key=True)\n",
+    "    name = Column(String)\n",
+    "    fullname = Column(String)\n",
+    "    nickname = Column(String)\n",
+    "\n",
+    "    def __repr__(self):\n",
+    "        return \"<User(name='%s', fullname='%s', nickname='%s')>\" % (\n",
+    "            self.name, self.fullname, self.nickname)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Table('users', MetaData(bind=None), Column('id', Integer(), table=<users>, primary_key=True, nullable=False), Column('name', String(), table=<users>), Column('fullname', String(), table=<users>), Column('nickname', String(), table=<users>), schema=None)"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "User.__table__ "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "2020-12-24 16:40:37,230 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1\n",
+      "2020-12-24 16:40:37,236 INFO sqlalchemy.engine.base.Engine ()\n",
+      "2020-12-24 16:40:37,239 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1\n",
+      "2020-12-24 16:40:37,240 INFO sqlalchemy.engine.base.Engine ()\n",
+      "2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine PRAGMA main.table_info(\"users\")\n",
+      "2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine ()\n",
+      "2020-12-24 16:40:37,244 INFO sqlalchemy.engine.base.Engine PRAGMA temp.table_info(\"users\")\n",
+      "2020-12-24 16:40:37,245 INFO sqlalchemy.engine.base.Engine ()\n",
+      "2020-12-24 16:40:37,246 INFO sqlalchemy.engine.base.Engine \n",
+      "CREATE TABLE users (\n",
+      "\tid INTEGER NOT NULL, \n",
+      "\tname VARCHAR, \n",
+      "\tfullname VARCHAR, \n",
+      "\tnickname VARCHAR, \n",
+      "\tPRIMARY KEY (id)\n",
+      ")\n",
+      "\n",
+      "\n",
+      "2020-12-24 16:40:37,247 INFO sqlalchemy.engine.base.Engine ()\n",
+      "2020-12-24 16:40:37,248 INFO sqlalchemy.engine.base.Engine COMMIT\n"
+     ]
+    }
+   ],
+   "source": [
+    "Base.metadata.create_all(engine)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ed\n",
+      "edsnickname\n",
+      "None\n"
+     ]
+    }
+   ],
+   "source": [
+    "ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname')\n",
+    "print(ed_user.name)\n",
+    "print(ed_user.nickname)\n",
+    "print(str(ed_user.id))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from sqlalchemy.orm import sessionmaker\n",
+    "Session = sessionmaker(bind=engine)\n",
+    "session = Session()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

config/environment.yml

@@ -7,6 +7,7 @@ dependencies:
   - black
   - isort
   - flake8
+  - jupyterlab
   - mypy
   - psycopg2
   - python-dotenv

config/models.py

@@ -0,0 +1,315 @@
+from datetime import datetime
+from sqlalchemy import Column, String, Boolean, \
+    Integer, SmallInteger, BigInteger, \
+    Float, TIMESTAMP, \
+    ForeignKey, UniqueConstraint
+from sqlalchemy.ext.declarative import declarative_base, declared_attr
+from sqlalchemy.dialects.postgresql import ARRAY as PSQL_ARRAY, \
+    JSONB as PSQL_JSONB, TEXT as PSQL_TEXT, INTEGER as PSQL_INTEGER
+from sqlalchemy.orm import relationship
+
+Base = declarative_base()
+
+
+class Participant(Base):
+    __tablename__ = "participants"
+    id = Column(Integer, primary_key=True)
+    username = Column(String(64), index=True, unique=True)
+    password_hash = Column(String(128))
+    token = Column(String(32), index=True, unique=True)
+    token_expiration_utc = Column(TIMESTAMP(timezone=False))
+
+    collection_start_utc = Column(TIMESTAMP(timezone=False))
+    last_upload_utc = Column(TIMESTAMP(timezone=False))
+    day_count_uploaded = Column(SmallInteger)
+    last_known_device_id = Column(String(length=36), nullable=False, default="")
+
+    active = Column(Boolean, index=True, nullable=False, default=False)
+    marked_active_utc = Column(TIMESTAMP(timezone=False))
+    day_count_active = Column(SmallInteger)
+
+    tester = Column(Boolean, index=True, nullable=False, default=False)
+
+    def __repr__(self):
+        return 'Participant()'
+
+    def __str__(self):
+        return '<Participant {}>'.format(self.username)
+
+
+class AWAREsensor(Base):
+    id = Column(BigInteger, primary_key=True, nullable=False)
+    _id = Column(BigInteger, nullable=False)
+    timestamp = Column(BigInteger, nullable=False)
+    device_id = Column(String(length=36), nullable=False)
+
+    @declared_attr
+    def __tablename__(cls):
+        return cls.__name__.lower()
+
+    @declared_attr
+    def participant_id(cls):
+        return Column(Integer, ForeignKey('participants.id'), nullable=False, index=True)
+
+    @declared_attr
+    def participants(cls):
+        return relationship('Participant', lazy="select", backref=cls.__tablename__)
+
+    @declared_attr
+    def __table_args__(cls):
+        return UniqueConstraint('device_id', '_id', name=cls.__tablename__ + "_twice"),
+    # I think it makes more sense to create a Constraint on device_id and _id rather than relate it to participant_id.
+    # _id is a primary key, auto incremented by AWARE.
+    # However, I would expect it to reset back to 1 if the application was reinstalled,
+    # similarly to how device_id resets.
+
+
+class Accelerometer(Base, AWAREsensor):
+    double_values_0 = Column(Float, nullable=False)
+    double_values_1 = Column(Float, nullable=False)
+    double_values_2 = Column(Float, nullable=False)
+    accuracy = Column(SmallInteger, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class GoogleAR(Base, AWAREsensor):
+    __tablename__ = "google_ar"
+    activity_name = Column(String(32), nullable=True)
+    activity_type = Column(SmallInteger, nullable=True)
+    confidence = Column(SmallInteger, nullable=True)
+    activities = Column(PSQL_JSONB(none_as_null=False), nullable=True)
+
+
+class Application(Base, AWAREsensor):
+    __tablename__ = "applications"
+#    package_name = Column(String, nullable=False)
+#    application_name = Column(String)
+    package_hash = Column(String(64), nullable=False)
+    play_store_genre = Column(String, nullable=True)
+    is_system_app = Column(Boolean)
+
+
+class Barometer(Base, AWAREsensor):
+    double_values_0 = Column(Float, nullable=False)
+    accuracy = Column(SmallInteger, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class BarometerSensor(Base, AWAREsensor):
+    __tablename__ = "barometer_sensor"
+    # Since this table is not really important, I will leave all columns as nullable. (nullable=True by default.)
+    double_sensor_maximum_range = Column(Float)
+    double_sensor_minimum_delay = Column(Float)
+    sensor_name = Column(String)
+    double_sensor_power_ma = Column(Float)
+    double_sensor_resolution = Column(Float)
+    sensor_type = Column(String)
+    sensor_vendor = Column(String)
+    sensor_version = Column(String)
+
+
+class Battery(Base, AWAREsensor):
+    battery_status = Column(SmallInteger, nullable=False)
+    battery_level = Column(SmallInteger, nullable=False)
+    battery_scale = Column(SmallInteger, nullable=False)
+    battery_voltage = Column(Integer, nullable=True)
+    battery_temperature = Column(Integer, nullable=True)
+    battery_adaptor = Column(SmallInteger, nullable=True)
+    battery_health = Column(SmallInteger, nullable=True)
+    battery_technology = Column(String, nullable=True)
+
+
+class Bluetooth(Base, AWAREsensor):
+    bt_address = Column(String(length=40), nullable=False)
+    bt_name = Column(String, nullable=True)
+    bt_rssi = Column(Integer, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class Call(Base, AWAREsensor):
+    call_type = Column(SmallInteger, nullable=False)
+    call_duration = Column(Integer, nullable=False)
+    trace = Column(String(length=40), nullable=True)
+
+
+class ESM(Base, AWAREsensor):
+    esm_status = Column(SmallInteger, nullable=False)
+    esm_user_answer = Column(String, nullable=True)
+    esm_notification_timeout = Column(SmallInteger)
+    esm_json = Column(PSQL_JSONB(none_as_null=False), nullable=False)
+    double_esm_user_answer_timestamp = Column(BigInteger, nullable=False)
+    esm_trigger = Column(String, nullable=True)
+    esm_session = Column(Integer, nullable=False)
+    esm_notification_id = Column(Integer, nullable=False)
+    esm_expiration_threshold = Column(SmallInteger)
+
+
+class Imperfection(Base):
+    id = Column(BigInteger, primary_key=True, nullable=False)
+    timestamp = Column(BigInteger, nullable=False)
+    error = Column(String)
+    data_type = Column(String)
+    data = Column(String)
+
+
+class LightSensor(Base, AWAREsensor):
+    __tablename__ = "light_sensor"
+    double_light_lux = Column(Float, nullable=False)
+    accuracy = Column(Integer, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class Location(Base, AWAREsensor):
+    __tablename__ = "locations"
+    double_latitude = Column(Float, nullable=False)
+    double_longitude = Column(Float, nullable=False)
+    double_bearing = Column(Float)
+    double_speed = Column(Float)
+    double_altitude = Column(Float)
+    provider = Column(String)
+    accuracy = Column(Integer)
+    label = Column(String, nullable=True)
+    category = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True)
+    category_short = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True)
+    distance = Column(PSQL_ARRAY(PSQL_INTEGER), nullable=True)
+    api_response_code = Column(SmallInteger, nullable=False)
+    api_response = Column(String, nullable=True)
+
+
+class Mic(Base, AWAREsensor):
+    half_s_speech = Column(Boolean, nullable=False)
+    rnnoutp = Column(Float, nullable=False)
+    flatness = Column(Float, nullable=False)
+    absmax = Column(Float, nullable=False)
+    max = Column(Float, nullable=False)
+    min = Column(Float, nullable=False)
+    # mfcc = Column(PSQL_ARRAY(PSQL_REAL, dimensions=13), nullable=False)
+    mfcc_0 = Column(Float, nullable=False)
+    mfcc_1 = Column(Float, nullable=False)
+    mfcc_2 = Column(Float, nullable=False)
+    mfcc_3 = Column(Float, nullable=False)
+    mfcc_4 = Column(Float, nullable=False)
+    mfcc_5 = Column(Float, nullable=False)
+    mfcc_6 = Column(Float, nullable=False)
+    mfcc_7 = Column(Float, nullable=False)
+    mfcc_8 = Column(Float, nullable=False)
+    mfcc_9 = Column(Float, nullable=False)
+    mfcc_10 = Column(Float, nullable=False)
+    mfcc_11 = Column(Float, nullable=False)
+    mfcc_12 = Column(Float, nullable=False)
+
+
+class Speech(Base, AWAREsensor):
+    speech_proportion = Column(Float, nullable=False)
+
+
+class NetworkData(Base, AWAREsensor):
+    __tablename__ = "network_data"
+    network_type = Column(SmallInteger, nullable=False)
+    network_subtype = Column(String(10), nullable=False)
+    network_state = Column(SmallInteger, nullable=True)
+
+
+class NetworkTraffic(Base, AWAREsensor):
+    __tablename__ = "network_traffic_data"
+    network_type = Column(SmallInteger, nullable=False)
+    double_received_bytes = Column(Float, nullable=True)
+    double_sent_bytes = Column(Float, nullable=True)
+    double_received_packets = Column(Float, nullable=True)
+    double_sent_packets = Column(Float, nullable=True)
+
+
+class Notification(Base, AWAREsensor):
+    __tablename__ = "notifications"
+#    package_name = Column(String, nullable=False)
+#    application_name = Column(String, nullable=False)
+    package_hash = Column(String(64), nullable=False)
+    play_store_genre = Column(String, nullable=True)
+    sound = Column(String, nullable=False)
+    vibrate = Column(String, nullable=False)
+    sound_app = Column(String, nullable=False)
+    vibrate_app = Column(String, nullable=False)
+    led_app = Column(String, nullable=False)
+    category = Column(String, nullable=False)
+    is_ongoing = Column(SmallInteger, nullable=False)
+    is_clearable = Column(SmallInteger, nullable=False)
+    is_group = Column(SmallInteger, nullable=True)
+
+
+class Processor(Base, AWAREsensor):
+    double_last_user = Column(Float, nullable=False)
+    double_last_system = Column(Float, nullable=False)
+    double_last_idle = Column(Float, nullable=False)
+    double_user_load = Column(Float, nullable=False)
+    double_system_load = Column(Float, nullable=False)
+    double_idle_load = Column(Float, nullable=False)
+
+
+class Proximity(Base, AWAREsensor):
+    double_proximity = Column(Float, nullable=False)
+    accuracy = Column(SmallInteger, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class Screen(Base, AWAREsensor):
+    screen_status = Column(SmallInteger)
+
+
+class SMS(Base, AWAREsensor):
+    message_type = Column(SmallInteger, nullable=False)
+    trace = Column(String(length=40), nullable=False)
+
+
+class Temperature(Base, AWAREsensor):
+    temperature_celsius = Column(Float, nullable=False)
+    accuracy = Column(SmallInteger, nullable=True)
+    label = Column(String, nullable=True)
+
+
+class TemperatureSensor(Base, AWAREsensor):
+    # I left all of these nullable, as we haven't seen any data from this sensor anyway.
+    __tablename__ = "temperature_sensor"
+    double_sensor_maximum_range = Column(Float)
+    double_sensor_minimum_delay = Column(Float)
+    sensor_name = Column(String)
+    double_sensor_power_ma = Column(Float)
+    double_sensor_resolution = Column(Float)
+    sensor_type = Column(String)
+    sensor_vendor = Column(String)
+    sensor_version = Column(String)
+
+
+class Timezone(Base, AWAREsensor):
+    timezone = Column(String, nullable=False)
+
+
+class WiFi(Base, AWAREsensor):
+    bssid = Column(String(length=40), nullable=False)
+    ssid = Column(String, nullable=True)
+    rssi = Column(Integer, nullable=False)
+    security = Column(String, nullable=True)
+    frequency = Column(Integer, nullable=True)
+    label = Column(String, nullable=True)
+
+
+all_AWARE_tables = [ESM, Location, Screen, LightSensor, Call, SMS, Application, Notification, Battery, WiFi, Proximity,
+                    Timezone, Processor, NetworkData, NetworkTraffic, Barometer, BarometerSensor, Temperature,
+                    TemperatureSensor, Bluetooth, Accelerometer, GoogleAR, Speech]
+
+all_AWARE_table_names = [table.__tablename__ for table in all_AWARE_tables]
+
+
+def increment_one(ii):
+    return ii + 1
+
+
+class AppCategories(Base):
+    __tablename__ = "app_categories"
+    id = Column(Integer, primary_key=True)
+    application_name = Column(String, nullable=True)
+    package_name = Column(String, nullable=False)
+    package_hash = Column(String(64), index=True, nullable=False, unique=True)
+    play_store_genre = Column(String, nullable=True)
+    play_store_response = Column(SmallInteger, nullable=False)
+    number_of_attempts = Column(SmallInteger, nullable=False, default=0, onupdate=increment_one)
+    last_attempt = Column(TIMESTAMP(timezone=False), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)

setup.py

@@ -1,6 +1,7 @@
 import os
 import sqlalchemy.engine.url
-
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
 from dotenv import load_dotenv
 
 load_dotenv()
@@ -15,3 +16,6 @@ db_uri = sqlalchemy.engine.url.URL(
     port=5432,
     database="staw"
 )
+
+db_engine = create_engine('sqlite:///:memory:', echo=True)
+Session = sessionmaker(bind=db_engine)