Lotta progress with historical import

jeffshek · Aug 19, 2020 · ac867d0 · ac867d0
1 parent 99fd38c
commit ac867d0
Show file tree

Hide file tree

Showing 7 changed files with 203 additions and 9 deletions.
diff --git a/open/core/betterself/models/supplement_log.py b/open/core/betterself/models/supplement_log.py
@@ -4,6 +4,7 @@
     CASCADE,
     CharField,
     DateTimeField,
+    PositiveIntegerField,
 )
 
 from open.core.betterself.constants import (
@@ -26,6 +27,7 @@ class SupplementLog(BaseModelWithUserGeneratedContent):
     # what time did the user take the five hour energy? use the time model
     # so eventually (maybe never) can do half-life analysis
     time = DateTimeField()
+    duration_minutes = PositiveIntegerField(null=True, blank=True)
 
     class Meta:
         unique_together = ("user", "time", "supplement")

diff --git a/open/core/betterself/serializers/supplement_log_serializers.py b/open/core/betterself/serializers/supplement_log_serializers.py
@@ -38,6 +38,7 @@ class Meta:
             "quantity",
             "time",
             "display_name",
+            "duration_minutes",
         )
 
     def get_display_name(self, instance):
@@ -75,6 +76,7 @@ class Meta:
             "source",
             "quantity",
             "time",
+            "duration_minutes",
         )
 
     def validate_supplement_uuid(self, value):

diff --git a/open/core/migrations/0020_supplementlog_duration_minutes.py b/open/core/migrations/0020_supplementlog_duration_minutes.py
@@ -0,0 +1,18 @@
+# Generated by Django 2.2.13 on 2020-08-19 15:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0019_dailyproductivitylog_mistakes"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="supplementlog",
+            name="duration_minutes",
+            field=models.PositiveIntegerField(blank=True, null=True),
+        ),
+    ]
diff --git a/open/core/scripts/betterself_import_legacy_app.py b/open/core/scripts/betterself_import_legacy_app.py
@@ -2,18 +2,21 @@
 From: Styled / copied from https://gist.github.com/kunanit/eb0723eef653788395bb41c661c1fa86
 Reason : This was used to import a legacy app from heroku onto GCP (this was a connection to postgresql database)
 
-Steps To Run
---
-pip install SQLAlchemy
 dpy runscript betterself_import_legacy_app
 """
-import ipdb
+from collections import defaultdict
+
 from ipdb import launch_ipdb_on_exception
 from sqlalchemy import create_engine
 from django.conf import settings
 import pandas as pd
 
+from open.core.betterself.models.daily_productivity_log import DailyProductivityLog
+from open.core.betterself.models.sleep_log import SleepLog
+from open.core.betterself.models.supplement import Supplement
+from open.core.betterself.models.supplement_log import SupplementLog
 from open.users.models import User
+from open.utilities.dataframes import change_dataframe_nans_to_none
 
 DATABASES = {
     "production": {
@@ -37,18 +40,55 @@
     database=db["NAME"],
 )
 
+local_store = defaultdict(lambda: None)
+
 
 def get_matching_row(df, column: str, matching_value):
+    if column == "id":
+        return df.loc[matching_value]
+
     result = df[column] == matching_value
     matched_df = df[result]
 
     assert len(matched_df) == 1
     return matched_df.iloc[0]
 
 
-def import_legacy_users():
-    engine = create_engine(engine_string)
+def get_matching_user(legacy_id):
+    engine = local_store["engine"]
+
+    if local_store["users_df"] is None:
+        users_df = pd.read_sql_table("users_user", engine, index_col="id")
+        local_store["users_df"] = users_df
+    else:
+        users_df = local_store["users_df"]
+
+    user_uuid = get_matching_row(users_df, "id", legacy_id)["uuid"]
+    user = User.objects.get(uuid=user_uuid)
+    return user
+
+
+def get_matching_supplement(legacy_id):
+    engine = local_store["engine"]
+
+    if local_store["supplements_df"] is None:
+        supplements_df = pd.read_sql_table(
+            "supplements_supplement", engine, index_col="id"
+        )
+        local_store["supplements_df"] = supplements_df
+    else:
+        supplements_df = local_store["supplements_df"]
+
+    instance_uuid = get_matching_row(supplements_df, "id", legacy_id)["uuid"]
+    instance = Supplement.objects.get(uuid=instance_uuid)
+    return instance
+
+
+def import_legacy_users(engine):
+    print("Importing Legacy Users")
     users_df = pd.read_sql_table("users_user", engine, index_col="id")
+    # avoid recursively having to get this over and over
+    local_store["users_df"] = users_df
 
     for index, legacy_user_details in users_df.iterrows():
 
@@ -83,8 +123,134 @@ def import_legacy_users():
         )
 
 
+def import_legacy_productivity(engine):
+    print("Importing Productivity")
+
+    productivity_df = pd.read_sql_table(
+        "events_dailyproductivitylog", engine, index_col="id"
+    )
+    productivity_df = change_dataframe_nans_to_none(productivity_df)
+
+    local_store["productivity_df"] = productivity_df
+
+    attributes_to_import = [
+        "uuid",
+        "source",
+        "date",
+        "very_productive_time_minutes",
+        "productive_time_minutes",
+        "neutral_time_minutes",
+        "distracting_time_minutes",
+        "very_distracting_time_minutes",
+    ]
+
+    for index, details in productivity_df.iterrows():
+        user = get_matching_user(details["user_id"])
+
+        defaults = {}
+        for attribute in attributes_to_import:
+            defaults[attribute] = details[attribute]
+
+        date = defaults.pop("date")
+        productivity_log, _ = DailyProductivityLog.objects.update_or_create(
+            user=user, date=date, defaults=defaults
+        )
+
+
+def import_legacy_sleep_log(engine):
+    print("Importing Sleep")
+
+    df = pd.read_sql_table("events_sleeplog", engine, index_col="id")
+    df = change_dataframe_nans_to_none(df)
+
+    attributes_to_import = [
+        "uuid",
+        "source",
+        "start_time",
+        "end_time",
+        "modified",
+    ]
+
+    for index, details in df.iterrows():
+        user = get_matching_user(details["user_id"])
+
+        defaults = {}
+        for attribute in attributes_to_import:
+            defaults[attribute] = details[attribute]
+
+        start_time = defaults.pop("start_time")
+        end_time = defaults.pop("end_time")
+
+        sleep_log, _ = SleepLog.objects.update_or_create(
+            user=user, start_time=start_time, end_time=end_time, defaults=defaults
+        )
+
+
+def import_legacy_supplements(engine):
+    print("Importing Supplements")
+
+    df = pd.read_sql_table("supplements_supplement", engine, index_col="id")
+    df = change_dataframe_nans_to_none(df)
+    local_store["supplements_df"] = df
+
+    attributes_to_import = [
+        "uuid",
+        "name",
+        "modified",
+    ]
+
+    for index, details in df.iterrows():
+        user = get_matching_user(details["user_id"])
+
+        defaults = {}
+        for attribute in attributes_to_import:
+            defaults[attribute] = details[attribute]
+
+        name = defaults.pop("name")
+        instance, _ = Supplement.objects.update_or_create(
+            user=user, name=name, defaults=defaults
+        )
+
+
+def import_legacy_supplements_log(engine):
+    print("Importing Supplements Log")
+
+    df = pd.read_sql_table("events_supplementlog", engine, index_col="id")
+    df = change_dataframe_nans_to_none(df)
+    local_store["supplement_log_df"] = df
+
+    attributes_to_import = [
+        "modified",
+        "uuid",
+        "source",
+        "quantity",
+        "time",
+        "duration_minutes",
+        "notes",
+    ]
+
+    for index, details in df.iterrows():
+        user = get_matching_user(details["user_id"])
+        supplement = get_matching_supplement(details["supplement_id"])
+
+        defaults = {}
+        for attribute in attributes_to_import:
+            defaults[attribute] = details[attribute]
+
+        instance, _ = SupplementLog.objects.update_or_create(
+            user=user, supplement=supplement, defaults=defaults
+        )
+
+
 def run_():
-    import_legacy_users()
+    engine = create_engine(engine_string)
+    local_store["engine"] = engine
+
+    import_legacy_users(engine)
+    import_legacy_productivity(engine)
+    import_legacy_sleep_log(engine)
+    import_legacy_supplements(engine)
+    import_legacy_supplements_log(engine)
 
 
 def run():

diff --git a/open/utilities/dataframes.py b/open/utilities/dataframes.py
@@ -0,0 +1,6 @@
+import pandas as pd
+
+
+def change_dataframe_nans_to_none(df):
+    df = df.where(pd.notnull(df), None)
+    return df
diff --git a/requirements/base.txt b/requirements/base.txt
@@ -17,6 +17,7 @@ cryptography==2.7
 daphne==2.5.0
 django==2.2.13  # https://www.djangoproject.com/
 django-allauth==0.42.0  # https://github.com/pennersr/django-allauth
+django-anymail[mailgun]==7.0.0  # https://github.com/anymail/django-anymail
 django-cors-headers==3.1.0
 django-crispy-forms==1.7.2  # https://github.com/django-crispy-forms/django-crispy-forms
 django-environ==0.4.5  # https://github.com/joke2k/django-environ
@@ -25,6 +26,7 @@ django-fsm==2.7.0
 django-model-utils==4.0.0  # https://github.com/jazzband/django-model-utils
 django-redis==4.12.1  # https://github.com/niwinz/django-redis
 django-rest-auth==0.9.5
+django-storages[google]==1.9.1  # https://github.com/jschneier/django-storages
 djangorestframework==3.11.0  # https://github.com/encode/django-rest-framework
 flower==0.9.3  # https://github.com/mher/flower
 hiredis==1.0.1

diff --git a/requirements/production.txt b/requirements/production.txt
@@ -1,10 +1,8 @@
 -r ./base.txt
 Collectfast==1.0.0  # https://github.com/antonagestam/collectfast
-django-anymail[mailgun]==7.0.0  # https://github.com/anymail/django-anymail
 
 # Django
 # ------------------------------------------------------------------------------
-django-storages[google]==1.9.1  # https://github.com/jschneier/django-storages
 
 gunicorn==20.0.4  # https://github.com/benoitc/gunicorn
 psycopg2==2.8.3 --no-binary psycopg2  # https://github.com/psycopg/psycopg2