Copy Models and edit for imports.

communication
junos 2020-12-24 16:46:55 +01:00
parent 8d114ebb13
commit bc27c71041
4 changed files with 496 additions and 1 deletions

View File

@ -0,0 +1,175 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'1.3.21'"
]
},
"execution_count": 1,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"import sqlalchemy\n",
"sqlalchemy.__version__ "
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"from sqlalchemy import create_engine\n",
"engine = create_engine('sqlite:///:memory:', echo=True)"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"from sqlalchemy.ext.declarative import declarative_base\n",
"Base = declarative_base()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"from sqlalchemy import Column, Integer, String\n",
"class User(Base):\n",
" __tablename__ = 'users'\n",
"\n",
" id = Column(Integer, primary_key=True)\n",
" name = Column(String)\n",
" fullname = Column(String)\n",
" nickname = Column(String)\n",
"\n",
" def __repr__(self):\n",
" return \"<User(name='%s', fullname='%s', nickname='%s')>\" % (\n",
" self.name, self.fullname, self.nickname)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Table('users', MetaData(bind=None), Column('id', Integer(), table=<users>, primary_key=True, nullable=False), Column('name', String(), table=<users>), Column('fullname', String(), table=<users>), Column('nickname', String(), table=<users>), schema=None)"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"User.__table__ "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"2020-12-24 16:40:37,230 INFO sqlalchemy.engine.base.Engine SELECT CAST('test plain returns' AS VARCHAR(60)) AS anon_1\n",
"2020-12-24 16:40:37,236 INFO sqlalchemy.engine.base.Engine ()\n",
"2020-12-24 16:40:37,239 INFO sqlalchemy.engine.base.Engine SELECT CAST('test unicode returns' AS VARCHAR(60)) AS anon_1\n",
"2020-12-24 16:40:37,240 INFO sqlalchemy.engine.base.Engine ()\n",
"2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine PRAGMA main.table_info(\"users\")\n",
"2020-12-24 16:40:37,242 INFO sqlalchemy.engine.base.Engine ()\n",
"2020-12-24 16:40:37,244 INFO sqlalchemy.engine.base.Engine PRAGMA temp.table_info(\"users\")\n",
"2020-12-24 16:40:37,245 INFO sqlalchemy.engine.base.Engine ()\n",
"2020-12-24 16:40:37,246 INFO sqlalchemy.engine.base.Engine \n",
"CREATE TABLE users (\n",
"\tid INTEGER NOT NULL, \n",
"\tname VARCHAR, \n",
"\tfullname VARCHAR, \n",
"\tnickname VARCHAR, \n",
"\tPRIMARY KEY (id)\n",
")\n",
"\n",
"\n",
"2020-12-24 16:40:37,247 INFO sqlalchemy.engine.base.Engine ()\n",
"2020-12-24 16:40:37,248 INFO sqlalchemy.engine.base.Engine COMMIT\n"
]
}
],
"source": [
"Base.metadata.create_all(engine)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ed\n",
"edsnickname\n",
"None\n"
]
}
],
"source": [
"ed_user = User(name='ed', fullname='Ed Jones', nickname='edsnickname')\n",
"print(ed_user.name)\n",
"print(ed_user.nickname)\n",
"print(str(ed_user.id))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"from sqlalchemy.orm import sessionmaker\n",
"Session = sessionmaker(bind=engine)\n",
"session = Session()"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.1"
}
},
"nbformat": 4,
"nbformat_minor": 4
}

View File

@ -7,6 +7,7 @@ dependencies:
- black - black
- isort - isort
- flake8 - flake8
- jupyterlab
- mypy - mypy
- psycopg2 - psycopg2
- python-dotenv - python-dotenv

315
config/models.py 100644
View File

@ -0,0 +1,315 @@
from datetime import datetime
from sqlalchemy import Column, String, Boolean, \
Integer, SmallInteger, BigInteger, \
Float, TIMESTAMP, \
ForeignKey, UniqueConstraint
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.dialects.postgresql import ARRAY as PSQL_ARRAY, \
JSONB as PSQL_JSONB, TEXT as PSQL_TEXT, INTEGER as PSQL_INTEGER
from sqlalchemy.orm import relationship
Base = declarative_base()
class Participant(Base):
__tablename__ = "participants"
id = Column(Integer, primary_key=True)
username = Column(String(64), index=True, unique=True)
password_hash = Column(String(128))
token = Column(String(32), index=True, unique=True)
token_expiration_utc = Column(TIMESTAMP(timezone=False))
collection_start_utc = Column(TIMESTAMP(timezone=False))
last_upload_utc = Column(TIMESTAMP(timezone=False))
day_count_uploaded = Column(SmallInteger)
last_known_device_id = Column(String(length=36), nullable=False, default="")
active = Column(Boolean, index=True, nullable=False, default=False)
marked_active_utc = Column(TIMESTAMP(timezone=False))
day_count_active = Column(SmallInteger)
tester = Column(Boolean, index=True, nullable=False, default=False)
def __repr__(self):
return 'Participant()'
def __str__(self):
return '<Participant {}>'.format(self.username)
class AWAREsensor(Base):
id = Column(BigInteger, primary_key=True, nullable=False)
_id = Column(BigInteger, nullable=False)
timestamp = Column(BigInteger, nullable=False)
device_id = Column(String(length=36), nullable=False)
@declared_attr
def __tablename__(cls):
return cls.__name__.lower()
@declared_attr
def participant_id(cls):
return Column(Integer, ForeignKey('participants.id'), nullable=False, index=True)
@declared_attr
def participants(cls):
return relationship('Participant', lazy="select", backref=cls.__tablename__)
@declared_attr
def __table_args__(cls):
return UniqueConstraint('device_id', '_id', name=cls.__tablename__ + "_twice"),
# I think it makes more sense to create a Constraint on device_id and _id rather than relate it to participant_id.
# _id is a primary key, auto incremented by AWARE.
# However, I would expect it to reset back to 1 if the application was reinstalled,
# similarly to how device_id resets.
class Accelerometer(Base, AWAREsensor):
double_values_0 = Column(Float, nullable=False)
double_values_1 = Column(Float, nullable=False)
double_values_2 = Column(Float, nullable=False)
accuracy = Column(SmallInteger, nullable=True)
label = Column(String, nullable=True)
class GoogleAR(Base, AWAREsensor):
__tablename__ = "google_ar"
activity_name = Column(String(32), nullable=True)
activity_type = Column(SmallInteger, nullable=True)
confidence = Column(SmallInteger, nullable=True)
activities = Column(PSQL_JSONB(none_as_null=False), nullable=True)
class Application(Base, AWAREsensor):
__tablename__ = "applications"
# package_name = Column(String, nullable=False)
# application_name = Column(String)
package_hash = Column(String(64), nullable=False)
play_store_genre = Column(String, nullable=True)
is_system_app = Column(Boolean)
class Barometer(Base, AWAREsensor):
double_values_0 = Column(Float, nullable=False)
accuracy = Column(SmallInteger, nullable=True)
label = Column(String, nullable=True)
class BarometerSensor(Base, AWAREsensor):
__tablename__ = "barometer_sensor"
# Since this table is not really important, I will leave all columns as nullable. (nullable=True by default.)
double_sensor_maximum_range = Column(Float)
double_sensor_minimum_delay = Column(Float)
sensor_name = Column(String)
double_sensor_power_ma = Column(Float)
double_sensor_resolution = Column(Float)
sensor_type = Column(String)
sensor_vendor = Column(String)
sensor_version = Column(String)
class Battery(Base, AWAREsensor):
battery_status = Column(SmallInteger, nullable=False)
battery_level = Column(SmallInteger, nullable=False)
battery_scale = Column(SmallInteger, nullable=False)
battery_voltage = Column(Integer, nullable=True)
battery_temperature = Column(Integer, nullable=True)
battery_adaptor = Column(SmallInteger, nullable=True)
battery_health = Column(SmallInteger, nullable=True)
battery_technology = Column(String, nullable=True)
class Bluetooth(Base, AWAREsensor):
bt_address = Column(String(length=40), nullable=False)
bt_name = Column(String, nullable=True)
bt_rssi = Column(Integer, nullable=True)
label = Column(String, nullable=True)
class Call(Base, AWAREsensor):
call_type = Column(SmallInteger, nullable=False)
call_duration = Column(Integer, nullable=False)
trace = Column(String(length=40), nullable=True)
class ESM(Base, AWAREsensor):
esm_status = Column(SmallInteger, nullable=False)
esm_user_answer = Column(String, nullable=True)
esm_notification_timeout = Column(SmallInteger)
esm_json = Column(PSQL_JSONB(none_as_null=False), nullable=False)
double_esm_user_answer_timestamp = Column(BigInteger, nullable=False)
esm_trigger = Column(String, nullable=True)
esm_session = Column(Integer, nullable=False)
esm_notification_id = Column(Integer, nullable=False)
esm_expiration_threshold = Column(SmallInteger)
class Imperfection(Base):
id = Column(BigInteger, primary_key=True, nullable=False)
timestamp = Column(BigInteger, nullable=False)
error = Column(String)
data_type = Column(String)
data = Column(String)
class LightSensor(Base, AWAREsensor):
__tablename__ = "light_sensor"
double_light_lux = Column(Float, nullable=False)
accuracy = Column(Integer, nullable=True)
label = Column(String, nullable=True)
class Location(Base, AWAREsensor):
__tablename__ = "locations"
double_latitude = Column(Float, nullable=False)
double_longitude = Column(Float, nullable=False)
double_bearing = Column(Float)
double_speed = Column(Float)
double_altitude = Column(Float)
provider = Column(String)
accuracy = Column(Integer)
label = Column(String, nullable=True)
category = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True)
category_short = Column(PSQL_ARRAY(PSQL_TEXT), nullable=True)
distance = Column(PSQL_ARRAY(PSQL_INTEGER), nullable=True)
api_response_code = Column(SmallInteger, nullable=False)
api_response = Column(String, nullable=True)
class Mic(Base, AWAREsensor):
half_s_speech = Column(Boolean, nullable=False)
rnnoutp = Column(Float, nullable=False)
flatness = Column(Float, nullable=False)
absmax = Column(Float, nullable=False)
max = Column(Float, nullable=False)
min = Column(Float, nullable=False)
# mfcc = Column(PSQL_ARRAY(PSQL_REAL, dimensions=13), nullable=False)
mfcc_0 = Column(Float, nullable=False)
mfcc_1 = Column(Float, nullable=False)
mfcc_2 = Column(Float, nullable=False)
mfcc_3 = Column(Float, nullable=False)
mfcc_4 = Column(Float, nullable=False)
mfcc_5 = Column(Float, nullable=False)
mfcc_6 = Column(Float, nullable=False)
mfcc_7 = Column(Float, nullable=False)
mfcc_8 = Column(Float, nullable=False)
mfcc_9 = Column(Float, nullable=False)
mfcc_10 = Column(Float, nullable=False)
mfcc_11 = Column(Float, nullable=False)
mfcc_12 = Column(Float, nullable=False)
class Speech(Base, AWAREsensor):
speech_proportion = Column(Float, nullable=False)
class NetworkData(Base, AWAREsensor):
__tablename__ = "network_data"
network_type = Column(SmallInteger, nullable=False)
network_subtype = Column(String(10), nullable=False)
network_state = Column(SmallInteger, nullable=True)
class NetworkTraffic(Base, AWAREsensor):
__tablename__ = "network_traffic_data"
network_type = Column(SmallInteger, nullable=False)
double_received_bytes = Column(Float, nullable=True)
double_sent_bytes = Column(Float, nullable=True)
double_received_packets = Column(Float, nullable=True)
double_sent_packets = Column(Float, nullable=True)
class Notification(Base, AWAREsensor):
__tablename__ = "notifications"
# package_name = Column(String, nullable=False)
# application_name = Column(String, nullable=False)
package_hash = Column(String(64), nullable=False)
play_store_genre = Column(String, nullable=True)
sound = Column(String, nullable=False)
vibrate = Column(String, nullable=False)
sound_app = Column(String, nullable=False)
vibrate_app = Column(String, nullable=False)
led_app = Column(String, nullable=False)
category = Column(String, nullable=False)
is_ongoing = Column(SmallInteger, nullable=False)
is_clearable = Column(SmallInteger, nullable=False)
is_group = Column(SmallInteger, nullable=True)
class Processor(Base, AWAREsensor):
double_last_user = Column(Float, nullable=False)
double_last_system = Column(Float, nullable=False)
double_last_idle = Column(Float, nullable=False)
double_user_load = Column(Float, nullable=False)
double_system_load = Column(Float, nullable=False)
double_idle_load = Column(Float, nullable=False)
class Proximity(Base, AWAREsensor):
double_proximity = Column(Float, nullable=False)
accuracy = Column(SmallInteger, nullable=True)
label = Column(String, nullable=True)
class Screen(Base, AWAREsensor):
screen_status = Column(SmallInteger)
class SMS(Base, AWAREsensor):
message_type = Column(SmallInteger, nullable=False)
trace = Column(String(length=40), nullable=False)
class Temperature(Base, AWAREsensor):
temperature_celsius = Column(Float, nullable=False)
accuracy = Column(SmallInteger, nullable=True)
label = Column(String, nullable=True)
class TemperatureSensor(Base, AWAREsensor):
# I left all of these nullable, as we haven't seen any data from this sensor anyway.
__tablename__ = "temperature_sensor"
double_sensor_maximum_range = Column(Float)
double_sensor_minimum_delay = Column(Float)
sensor_name = Column(String)
double_sensor_power_ma = Column(Float)
double_sensor_resolution = Column(Float)
sensor_type = Column(String)
sensor_vendor = Column(String)
sensor_version = Column(String)
class Timezone(Base, AWAREsensor):
timezone = Column(String, nullable=False)
class WiFi(Base, AWAREsensor):
bssid = Column(String(length=40), nullable=False)
ssid = Column(String, nullable=True)
rssi = Column(Integer, nullable=False)
security = Column(String, nullable=True)
frequency = Column(Integer, nullable=True)
label = Column(String, nullable=True)
all_AWARE_tables = [ESM, Location, Screen, LightSensor, Call, SMS, Application, Notification, Battery, WiFi, Proximity,
Timezone, Processor, NetworkData, NetworkTraffic, Barometer, BarometerSensor, Temperature,
TemperatureSensor, Bluetooth, Accelerometer, GoogleAR, Speech]
all_AWARE_table_names = [table.__tablename__ for table in all_AWARE_tables]
def increment_one(ii):
return ii + 1
class AppCategories(Base):
__tablename__ = "app_categories"
id = Column(Integer, primary_key=True)
application_name = Column(String, nullable=True)
package_name = Column(String, nullable=False)
package_hash = Column(String(64), index=True, nullable=False, unique=True)
play_store_genre = Column(String, nullable=True)
play_store_response = Column(SmallInteger, nullable=False)
number_of_attempts = Column(SmallInteger, nullable=False, default=0, onupdate=increment_one)
last_attempt = Column(TIMESTAMP(timezone=False), nullable=False, default=datetime.utcnow, onupdate=datetime.utcnow)

View File

@ -1,6 +1,7 @@
import os import os
import sqlalchemy.engine.url import sqlalchemy.engine.url
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
@ -15,3 +16,6 @@ db_uri = sqlalchemy.engine.url.URL(
port=5432, port=5432,
database="staw" database="staw"
) )
db_engine = create_engine('sqlite:///:memory:', echo=True)
Session = sessionmaker(bind=db_engine)