Merge pull request #2 from TogetherCrew/feat/CI

feat: Added our CI worflow to github actions!

.github/workflows/production.yml

@@ -0,0 +1,12 @@
+name: Production CI/CD Pipeline
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  ci:
+    uses: TogetherCrew/operations/.github/workflows/ci.yml@main
+    secrets:
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

.github/workflows/start.staging.yml

@@ -0,0 +1,9 @@
+name: Staging CI/CD Pipeline
+
+on: pull_request
+
+jobs:
+  ci:
+    uses: TogetherCrew/operations/.github/workflows/ci.yml@main
+    secrets:
+      CC_TEST_REPORTER_ID: ${{ secrets.CC_TEST_REPORTER_ID }}

app.py

@@ -1,14 +1,8 @@
+import logging
 from datetime import datetime
-from pandas.api.types import (
-    is_categorical_dtype,
-    is_datetime64_any_dtype,
-    is_numeric_dtype,
-    is_object_dtype,
-)
+
 import pandas as pd
 import streamlit as st
-import logging
-
 from utils.mongo import MongoSingleton
 from utils.process_guild_data import process_guild_data
 
@@ -43,6 +37,6 @@ def load_guilds_latest_date_df():
 logging.basicConfig(level=logging.INFO)
 
 st.subheader("MongoDB data Analytics")
-df = load_guilds_latest_date_df()
+load_guilds_latest_date_df()
 # df = process_df(df)
 # st.dataframe(df, use_container_width=True, hide_index=True)

tests/integration/test_credentials_load.py

@@ -0,0 +1,14 @@
+from unittest import TestCase
+
+from utils.creds import get_mongo_credentials
+
+
+class TestLoadCredentials(TestCase):
+    def test_creds_non_empty(self):
+        creds = get_mongo_credentials()
+
+        self.assertIsNotNone(creds["user"])
+        self.assertIsNotNone(creds["password"])
+        self.assertIsNotNone(creds["host"])
+        self.assertIsNotNone(creds["port"])
+        self.assertNotEqual(creds["connection_str"], "mongodb://None:None@None:None")

tests/unit/test_sample.py

@@ -3,4 +3,4 @@
 
 class TestSample(TestCase):
     def test_sample(self):
-        self.assertEqual(True, True)
+        self.assertEqual(True, True)

utils/creds.py

@@ -1,6 +1,7 @@
-from dotenv import load_dotenv
 import os
 
+from dotenv import load_dotenv
+
 
 def get_mongo_credentials():
     """

utils/mongo_base.py

@@ -24,7 +24,6 @@ def get_guild_members_count(self) -> int:
     def get_latest_document(
         self, db_name: str, collection_name: str, date_field: str, **kwargs
     ) -> dict[str, Any]:
-
         filters = kwargs.get("filters", None)
         client = MongoSingleton.get_instance().client
         if filters is None:

utils/mongo_utils.py

@@ -1,12 +1,13 @@
 from datetime import datetime
+
 from utils.mongo_base import MongoBase
 
 
 class MongoUtils(MongoBase):
     def __init__(self, guild_id: str) -> None:
         super().__init__(guild_id)
 
-    def get_latest_discord_raw_info_date(self) -> datetime:
+    def get_latest_discord_raw_info_date(self) -> datetime | None:
         date_field = "createdDate"
         latest_document = self.get_latest_document(
             db_name=self.guild_id, collection_name="rawinfos", date_field=date_field
@@ -20,18 +21,18 @@ def get_latest_memberactivities_date(self) -> str:
             collection_name="memberactivities",
             date_field=date_field,
         )
-        return self.get_latest_date(latest_document, date_field)
+        return self.get_latest_date(latest_document, date_field)  # type: ignore
 
     def get_latest_heatmaps_date(self) -> str:
         date_field = "date"
         latest_document = self.get_latest_document(
             db_name=self.guild_id, collection_name="heatmaps", date_field=date_field
         )
-        return self.get_latest_date(latest_document, date_field)
+        return self.get_latest_date(latest_document, date_field)  # type: ignore
 
     def get_latest_fired_saga(
         self, guild_id: str | None = None, platform_id: str | None = None
-    ) -> datetime:
+    ) -> datetime | None:
         date_field = "createdAt"
         if platform_id:
             latest_document = self.get_latest_document(
@@ -44,7 +45,7 @@ def get_latest_fired_saga(
                 },
             )
         elif guild_id:
-            fetched_platform_id = self.get_guild_platform_id(guild_id)
+            fetched_platform_id = self.get_guild_platform_id()
             latest_document = self.get_latest_document(
                 db_name="Saga",
                 collection_name="sagas",

utils/process_guild_data.py

@@ -3,9 +3,12 @@
 from utils.mongo_utils import MongoUtils
 
 
-def process_guild_data(platform_document: dict) -> dict[str, str | datetime | None]:
-    data: dict[str, str | datetime | None] = {}
-
+def process_guild_data(
+    platform_document: dict,
+) -> dict[str, str | int | datetime | None]:
+    """
+    process each platform and prepare the analytics for it
+    """
     platform_id = str(platform_document["_id"])
     guild_id = platform_document["metadata"]["id"]
     utils = MongoUtils(guild_id)
@@ -18,12 +21,6 @@ def process_guild_data(platform_document: dict) -> dict[str, str | datetime | No
     else:
         selected_channel_count = -1
 
-    data["guild_id"] = guild_id
-    data["guild_name"] = guild_name
-    data["connected_at"] = connected_at
-    data["platform_id"] = platform_id
-    data["disconnected_at"] = disconnected_at
-
     # getting the latest dates
     raw_infos_date = utils.get_latest_discord_raw_info_date()
     fired_sage_date = utils.get_latest_fired_saga(platform_id=platform_id)
@@ -37,13 +34,20 @@ def process_guild_data(platform_document: dict) -> dict[str, str | datetime | No
         from_date=datetime.now() - timedelta(days=31)
     )
 
-    data["selected_channels_count"] = selected_channel_count
-    data["extracted_channel_counts"] = len(extracted_channels)
-    data["latest_raw_info"] = raw_infos_date
-    data["latest_analyzer_run_fired_saga"] = fired_sage_date
-    data["latest_heatmaps"] = heatmaps_date
-    data["latest_memberactivities"] = memberactivities_date
-    data["raw_data_count_30days"] = raw_data_count
-    data["guild_members_count"] = guild_members_count
+    data: dict[str, str | int | datetime | None] = {
+        "guild_id": guild_id,
+        "guild_name": guild_name,
+        "connected_at": connected_at,
+        "platform_id": platform_id,
+        "disconnected_at": disconnected_at,
+        "selected_channels_count": selected_channel_count,
+        "extracted_channel_counts": len(extracted_channels),
+        "latest_raw_info": raw_infos_date,
+        "latest_analyzer_run_fired_saga": fired_sage_date,
+        "latest_heatmaps": heatmaps_date,
+        "latest_memberactivities": memberactivities_date,
+        "raw_data_count_30days": raw_data_count,
+        "guild_members_count": guild_members_count,
+    }
 
     return data