Add MongoDB query utilities and update batch size for document insertion

db/query_utils.py

@@ -0,0 +1,270 @@
+"""MongoDB query utilities for filtering documents by update timestamp."""
+
+__author__ = 'Wendell Jones'
+
+from typing import Any, Dict, List, Optional
+from datetime import datetime, timedelta, timezone
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class MongoQueryUtils:
+    """Utilities for querying MongoDB collections."""
+
+    @staticmethod
+    def get_recent_updates(
+        collection: Any,
+        minutes: int = 60,
+        updated_field: str = 'updated',
+        query_filter: Optional[Dict] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Find documents updated in the last N minutes.
+        
+        Handles both epoch timestamps (int/float) and ISO datetime strings.
+        
+        Args:
+            collection: PyMongo collection object.
+            minutes: Number of minutes to look back (default: 60).
+            updated_field: Name of the updated timestamp field (default: 'updated').
+            query_filter: Additional MongoDB query filter to apply (optional).
+            
+        Returns:
+            List of documents updated within the time window.
+            
+        Example:
+            from db.query_utils import MongoQueryUtils
+            
+            recent = MongoQueryUtils.get_recent_updates(
+                collection=mongo_db['series'],
+                minutes=60,
+                updated_field='updated',
+            )
+            print(f"Found {len(recent)} recently updated documents")
+        """
+        if query_filter is None:
+            query_filter = {}
+        
+        # Calculate cutoff time (60 minutes ago)
+        now = datetime.now(timezone.utc)
+        cutoff_time = now - timedelta(minutes=minutes)
+        cutoff_epoch = int(cutoff_time.timestamp())
+        
+        logger.info(
+            f"Searching for documents updated in last {minutes} minutes "
+            f"(since {cutoff_time.isoformat()})"
+        )
+        
+        # Build query for documents updated after cutoff
+        # Support both epoch timestamps and ISO datetime strings
+        query = {
+            **query_filter,
+            '$or': [
+                # Epoch timestamp (int/float)
+                {updated_field: {'$gte': cutoff_epoch}},
+                # ISO datetime string
+                {updated_field: {'$gte': cutoff_time.isoformat()}},
+            ]
+        }
+        
+        documents = list(collection.find(query))
+        logger.info(f"Found {len(documents)} documents matching criteria")
+        
+        return documents
+
+    @staticmethod
+    def get_recent_updates_by_id(
+        collection: Any,
+        minutes: int = 60,
+        updated_field: str = 'updated',
+        id_field: str = 'id',
+        query_filter: Optional[Dict] = None,
+    ) -> List[int]:
+        """
+        Get IDs of documents updated in the last N minutes.
+        
+        Useful for batch processing or sync operations.
+        
+        Args:
+            collection: PyMongo collection object.
+            minutes: Number of minutes to look back (default: 60).
+            updated_field: Name of the updated timestamp field (default: 'updated').
+            id_field: Name of the ID field (default: 'id').
+            query_filter: Additional MongoDB query filter (optional).
+            
+        Returns:
+            List of document IDs that were recently updated.
+            
+        Example:
+            ids = MongoQueryUtils.get_recent_updates_by_id(
+                collection=mongo_db['series'],
+                minutes=60,
+            )
+            print(f"Series IDs to update: {ids}")
+        """
+        docs = MongoQueryUtils.get_recent_updates(
+            collection=collection,
+            minutes=minutes,
+            updated_field=updated_field,
+            query_filter=query_filter,
+        )
+        
+        ids = [doc.get(id_field) for doc in docs if id_field in doc]
+        return ids
+
+    @staticmethod
+    def get_updates_since(
+        collection: Any,
+        since_timestamp: Optional[Any] = None,
+        minutes: int = 60,
+        updated_field: str = 'updated',
+        query_filter: Optional[Dict] = None,
+    ) -> List[Dict[str, Any]]:
+        """
+        Find documents updated since a specific timestamp or N minutes ago.
+        
+        Args:
+            collection: PyMongo collection object.
+            since_timestamp: Epoch int, float, or ISO datetime string (optional).
+                            If None, defaults to N minutes ago (see minutes parameter).
+            minutes: If since_timestamp is None, look back this many minutes (default: 60).
+            updated_field: Name of the updated timestamp field (default: 'updated').
+            query_filter: Additional MongoDB query filter (optional).
+            
+        Returns:
+            List of documents updated since the timestamp.
+            
+        Example:
+            # Get updates since a specific timestamp
+            cutoff = 1704067200  # Epoch timestamp
+            docs = MongoQueryUtils.get_updates_since(
+                collection=mongo_db['series'],
+                since_timestamp=cutoff,
+            )
+            
+            # Or use default 60 minutes ago
+            docs = MongoQueryUtils.get_updates_since(
+                collection=mongo_db['series'],
+            )
+            
+            # Or specify different time window
+            docs = MongoQueryUtils.get_updates_since(
+                collection=mongo_db['series'],
+                minutes=120,  # Last 2 hours
+            )
+        """
+        if query_filter is None:
+            query_filter = {}
+        
+        # If no timestamp provided, use minutes parameter to calculate cutoff
+        if since_timestamp is None:
+            now = datetime.now(timezone.utc)
+            cutoff_time = now - timedelta(minutes=minutes)
+            since_timestamp = int(cutoff_time.timestamp())
+            logger.info(f"Using default {minutes} minutes ago: {cutoff_time.isoformat()}")
+        
+        query = {
+            **query_filter,
+            '$or': [
+                {updated_field: {'$gte': since_timestamp}},
+                {updated_field: {'$gte': str(since_timestamp)}},
+            ]
+        }
+        
+        documents = list(collection.find(query))
+        logger.info(f"Found {len(documents)} documents updated since {since_timestamp}")
+        
+        return documents
+
+    @staticmethod
+    def count_recent_updates(
+        collection: Any,
+        minutes: int = 60,
+        updated_field: str = 'updated',
+        query_filter: Optional[Dict] = None,
+    ) -> int:
+        """
+        Count documents updated in the last N minutes.
+        
+        Args:
+            collection: PyMongo collection object.
+            minutes: Number of minutes to look back (default: 60).
+            updated_field: Name of the updated timestamp field (default: 'updated').
+            query_filter: Additional MongoDB query filter (optional).
+            
+        Returns:
+            Count of recently updated documents.
+            
+        Example:
+            count = MongoQueryUtils.count_recent_updates(
+                collection=mongo_db['series'],
+                minutes=60,
+            )
+            print(f"{count} series were updated in the last hour")
+        """
+        if query_filter is None:
+            query_filter = {}
+        
+        now = datetime.now(timezone.utc)
+        cutoff_time = now - timedelta(minutes=minutes)
+        cutoff_epoch = int(cutoff_time.timestamp())
+        
+        query = {
+            **query_filter,
+            '$or': [
+                {updated_field: {'$gte': cutoff_epoch}},
+                {updated_field: {'$gte': cutoff_time.isoformat()}},
+            ]
+        }
+        
+        count = collection.count_documents(query)
+        return count
+
+
+# Convenience functions
+def get_recent_updates(
+    collection: Any,
+    minutes: int = 60,
+    updated_field: str = 'updated',
+    query_filter: Optional[Dict] = None,
+) -> List[Dict[str, Any]]:
+    """Find documents updated in the last N minutes."""
+    return MongoQueryUtils.get_recent_updates(
+        collection=collection,
+        minutes=minutes,
+        updated_field=updated_field,
+        query_filter=query_filter,
+    )
+
+
+def get_recent_ids(
+    collection: Any,
+    minutes: int = 60,
+    updated_field: str = 'updated',
+    id_field: str = 'id',
+    query_filter: Optional[Dict] = None,
+) -> List[int]:
+    """Get IDs of documents updated in the last N minutes."""
+    return MongoQueryUtils.get_recent_updates_by_id(
+        collection=collection,
+        minutes=minutes,
+        updated_field=updated_field,
+        id_field=id_field,
+        query_filter=query_filter,
+    )
+
+
+def count_recent(
+    collection: Any,
+    minutes: int = 60,
+    updated_field: str = 'updated',
+    query_filter: Optional[Dict] = None,
+) -> int:
+    """Count documents updated in the last N minutes."""
+    return MongoQueryUtils.count_recent_updates(
+        collection=collection,
+        minutes=minutes,
+        updated_field=updated_field,
+        query_filter=query_filter,
+    )

db/schema_generator.py

@@ -335,7 +335,7 @@ def create_postgres_table_from_mongo(
 class MongoDocumentInserter:
     """Insert MongoDB documents into PostgreSQL tables with type conversion."""
 
-    def __init__(self, batch_size: int = 1000):
+    def __init__(self, batch_size: int = 10000):
         """
         Initialize the document inserter.
         

logs/Logger.py

@@ -44,12 +44,12 @@ class LPrint:
         """Check the log file size and rotate if necessary."""
         log_size = os.path.getsize(self.logfile)
         print(f'Logfile Size: {log_size} ({self.humanbytes(log_size)})')
-        if force_rotate or log_size > 10_000_000:  # 10 MB
+        if force_rotate or log_size > 50_000_000:  # 10 MB
             self.rotate_logs()
 
     def rotate_logs(self):
         """Rotate the log file."""
-        from logrotater import LogRotate  # Assuming logrotater is a custom module
+        from logs.logrotater import LogRotate  # Assuming logrotater is a custom module
         rotater = LogRotate(prefix=self.logfile, verbose=True)
         try:
             rotater.rotate()

logs/logrotater.py

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+
+''' A python logrotate library'''
+
+import os
+import glob
+
+__author__ = 'Kyle Laplante'
+__copyright__ = 'Copyright 2013, Kyle Laplante'
+__date__ = '07-07-2013'
+
+__license__ = 'GPL'
+__version__ = '1.3'
+__email__ = 'kyle.laplante@gmail.com'
+
+
+class LogRotateException(Exception):
+    pass
+
+
+class LogRotate:
+
+    """This library is a simple logrotater (or file rotater).
+Simply pass in the path to the main logfile and this
+library will rotate all the logs by an increment of 1.
+Example:
+
+import logrotater
+
+rotater = logrotater.LogRotate(prefix='/home/kyle/p4.log', verbose=True)
+
+rotater.rotate()
+
+The prefix path should be the path of the main logfile without the .N extension.
+The previous example would rotate /home/kyle/p4.log.N to /home/kyle/p4.log.N+1,
+move /home/kyle/p4.log to /home/kyle/p4.log.1 and create a new empty /home/kyle/p4.log."""
+
+    def __init__(self, *args, **kwargs):
+
+        '''Required kwargs: prefix="some path"
+Optional kwargs: verbose=True
+If verbose is True then LogRotate will print what it is doing with each file.'''
+
+        self.verbose = kwargs.get('verbose', False)
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        if not os.path.isfile(self.prefix):
+            raise LogRotateException("Must use a valid prefix filename")
+
+    def rotate(self, *args, **kwargs):
+        self.files = glob.glob(self.prefix + "*")
+        if not len(self.files) > 0:
+            raise LogRotateException(
+                "No files with the prefix %s found" % self.prefix)
+        self.nums = []
+        for file in self.files:
+            try:
+                self.nums.append(int(file.split(self.prefix + '.')[-1]))
+            except ValueError:
+                pass
+        self.nums.sort()
+        self.nums.reverse()
+        for num in self.nums:
+            new_num = num + 1
+            if self.verbose:
+                print("moving %s.%d to %s.%d" % (self.prefix, num, self.prefix, new_num))
+            os.rename("%s.%d" % (self.prefix, num), "%s.%d" %
+                      (self.prefix, new_num))
+        if self.verbose:
+            print("moving %s to %s.1" % (self.prefix, self.prefix))
+        os.rename(self.prefix, self.prefix + '.1')
+        self.touch(self.prefix)
+
+    def touch(self, path):
+
+        '''a simple method to touch a file'''
+
+        if self.verbose:
+            print("Creating file: %s" % path)
+        f = open(path, 'a')
+        os.utime(path, None)
+
+if __name__ == '__main__':
+
+    from optparse import OptionParser
+    import sys
+
+    usage = '''%prog [options] <file prefix>
+
+File prefix is the path to the main logfile.'''
+
+    parser = OptionParser(usage=usage)
+    parser.add_option(
+        '-p', '--prefix', dest='prefix', default=None, help='Filename prefix.')
+    parser.add_option('-v', '--verbose', dest='verbose',
+                      action='store_true', default=False, help='Enable verbose output')
+    (opts, args) = parser.parse_args()
+
+    if opts.prefix is None:
+        print("Please enter a filename prefix")
+        sys.exit(1)
+
+    try:
+        rotater = LogRotate(prefix=opts.prefix, verbose=opts.verbose)
+    except LogRotateException as e:
+        print(e)
+        sys.exit(1)
+    try:
+        rotater.rotate()
+    except LogRotateException as e:
+        print(e)
+        sys.exit(1)

mongodb2postgres.py

@@ -73,7 +73,7 @@ for tableName in tableNames:
 
 
 
-inserter = MongoDocumentInserter(batch_size=1000)
+inserter = MongoDocumentInserter(batch_size=10000)
 
 # Map table names to their MongoDB collections and primary keys
 collection_map = {
@@ -118,3 +118,23 @@ for tableName in tableNames:
 # if __name__ == "__main__":
 #     sys.exit(main())
 
+# Below are the relevant methods from db/functions.py and db/query_utils.py that are used in the above code. They are included here for completeness and context.
+# Needs to be incorporated into logic so that only releveant documents are updated.
+
+# from db.query_utils import MongoQueryUtils
+
+# # With additional filters
+# recent = MongoQueryUtils.get_recent_updates(
+#     collection=mongo_updater.mgseries,
+#     minutes=60,
+#     updated_field='updated',
+#     query_filter={'status': 'Ended'},  # Only ended shows
+# )
+
+# # Since a specific timestamp
+# docs = MongoQueryUtils.get_updates_since(
+#     collection=mongo_updater.mgseries,
+#     since_timestamp=None,  # Epoch or ISO string
+# )
+# for doc in docs:
+#     print(doc)