From 046a26541bab460f162b7858b1c353f9d4933c40 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 20 Feb 2024 17:39:00 -0500 Subject: [PATCH 01/11] Memory trace for DataCacheUpdate CP thread --- .../WMStats/CherryPyThreads/DataCacheUpdate.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index 1cdff6d476..384e851eb8 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -1,6 +1,7 @@ from __future__ import (division, print_function) import time +import tracemalloc from WMCore.REST.CherryPyPeriodicTask import CherryPyPeriodicTask from WMCore.WMStats.DataStructs.DataCache import DataCache from WMCore.Services.WMStats.WMStatsReader import WMStatsReader @@ -12,6 +13,7 @@ def __init__(self, rest, config): self.getJobInfo = getattr(config, "getJobInfo", False) super(DataCacheUpdate, self).__init__(config) + tracemalloc.start() def setConcurrentTasks(self, config): """ @@ -27,6 +29,7 @@ def gatherActiveDataStats(self, config): try: tStart = time.time() if DataCache.islatestJobDataExpired(): + snapshot1 = tracemalloc.take_snapshot() wmstatsDB = WMStatsReader(config.wmstats_url, reqdbURL=config.reqmgrdb_url, reqdbCouchApp="ReqMgr", logger=self.logger) self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) @@ -34,9 +37,21 @@ def gatherActiveDataStats(self, config): self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO) tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False) jobData.update(tempData) + snapshot2 = tracemalloc.take_snapshot() self.logger.info("Running setlatestJobData...") DataCache.setlatestJobData(jobData) self.logger.info("DataCache is up-to-date with %d requests data", len(jobData)) + snapshot3 = tracemalloc.take_snapshot() + + self.logger.info("Memory stats between snapshot2 and snapshot1") + top_stats = snapshot2.compare_to(snapshot1, 'lineno') + for thisStat in top_stats[:5]: + self.logger.info(thisStat) + + self.logger.info("Memory stats between snapshot3 and snapshot2") + top_stats = snapshot3.compare_to(snapshot2, 'lineno') + for thisStat in top_stats[:5]: + self.logger.info(thisStat) except Exception as ex: self.logger.exception("Exception updating DataCache. Error: %s", str(ex)) self.logger.info("Total time loading data from ReqMgr2 and WMStats: %s", time.time() - tStart) From 88f94970a973ebb946424e8463116ad792bc5615 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 20 Feb 2024 18:00:09 -0500 Subject: [PATCH 02/11] dont make diff of snapshots --- .../CherryPyThreads/DataCacheUpdate.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index 384e851eb8..65d2240120 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -29,7 +29,12 @@ def gatherActiveDataStats(self, config): try: tStart = time.time() if DataCache.islatestJobDataExpired(): - snapshot1 = tracemalloc.take_snapshot() + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('lineno') + self.logger.info("Snapshot memory stats:") + for thisStat in top_stats[:5]: + self.logger.info(thisStat) + wmstatsDB = WMStatsReader(config.wmstats_url, reqdbURL=config.reqmgrdb_url, reqdbCouchApp="ReqMgr", logger=self.logger) self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) @@ -37,21 +42,9 @@ def gatherActiveDataStats(self, config): self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO) tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False) jobData.update(tempData) - snapshot2 = tracemalloc.take_snapshot() self.logger.info("Running setlatestJobData...") DataCache.setlatestJobData(jobData) self.logger.info("DataCache is up-to-date with %d requests data", len(jobData)) - snapshot3 = tracemalloc.take_snapshot() - - self.logger.info("Memory stats between snapshot2 and snapshot1") - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - for thisStat in top_stats[:5]: - self.logger.info(thisStat) - - self.logger.info("Memory stats between snapshot3 and snapshot2") - top_stats = snapshot3.compare_to(snapshot2, 'lineno') - for thisStat in top_stats[:5]: - self.logger.info(thisStat) except Exception as ex: self.logger.exception("Exception updating DataCache. Error: %s", str(ex)) self.logger.info("Total time loading data from ReqMgr2 and WMStats: %s", time.time() - tStart) From 3477e1db40ce9a9f4a2df8daa0dc6ddec9e98c74 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 20 Feb 2024 18:23:08 -0500 Subject: [PATCH 03/11] use memory_profiler instead --- .../WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index 65d2240120..05c3831130 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -1,7 +1,7 @@ from __future__ import (division, print_function) import time -import tracemalloc +from memory_profiler import profile from WMCore.REST.CherryPyPeriodicTask import CherryPyPeriodicTask from WMCore.WMStats.DataStructs.DataCache import DataCache from WMCore.Services.WMStats.WMStatsReader import WMStatsReader @@ -21,6 +21,7 @@ def setConcurrentTasks(self, config): """ self.concurrentTasks = [{'func': self.gatherActiveDataStats, 'duration': 300}] + @profile def gatherActiveDataStats(self, config): """ gather active data statistics @@ -29,12 +30,6 @@ def gatherActiveDataStats(self, config): try: tStart = time.time() if DataCache.islatestJobDataExpired(): - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('lineno') - self.logger.info("Snapshot memory stats:") - for thisStat in top_stats[:5]: - self.logger.info(thisStat) - wmstatsDB = WMStatsReader(config.wmstats_url, reqdbURL=config.reqmgrdb_url, reqdbCouchApp="ReqMgr", logger=self.logger) self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) From ae82ed80e77205a6f335494d9502af4f5802eaaf Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 20 Feb 2024 18:24:57 -0500 Subject: [PATCH 04/11] remove tracemalloc --- src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index 05c3831130..19bf868a7d 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -11,9 +11,7 @@ class DataCacheUpdate(CherryPyPeriodicTask): def __init__(self, rest, config): self.getJobInfo = getattr(config, "getJobInfo", False) - super(DataCacheUpdate, self).__init__(config) - tracemalloc.start() def setConcurrentTasks(self, config): """ From 3c915d90e77ccd8c6284417f28fdf914e2bb5e40 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Tue, 20 Feb 2024 20:00:45 -0500 Subject: [PATCH 05/11] profile requestcache --- src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py | 3 +-- src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py | 2 ++ 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index 19bf868a7d..c0a3fc9829 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -33,8 +33,7 @@ def gatherActiveDataStats(self, config): self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) jobData = wmstatsDB.getActiveData(WMSTATS_JOB_INFO, jobInfoFlag=self.getJobInfo) self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO) - tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False) - jobData.update(tempData) + jobData.update(wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False)) self.logger.info("Running setlatestJobData...") DataCache.setlatestJobData(jobData) self.logger.info("DataCache is up-to-date with %d requests data", len(jobData)) diff --git a/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py b/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py index 829f387c06..7b77dee7f6 100644 --- a/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py +++ b/src/python/WMCore/WMStats/Service/ActiveRequestJobInfo.py @@ -3,6 +3,7 @@ Just wait for the server cache to be updated """ from __future__ import (division, print_function) +from memory_profiler import profile from WMCore.REST.Server import RESTEntity, restcall, rows from WMCore.REST.Tools import tools from WMCore.REST.Error import DataCacheEmpty @@ -25,6 +26,7 @@ def validate(self, apiobj, method, api, param, safe): @restcall(formats=[('text/plain', PrettyJSONFormat()), ('application/json', JSONFormat())]) @tools.expires(secs=-1) + @profile() def get(self): # This assumes DataCahe is periodically updated. # If data is not updated, need to check, dataCacheUpdate log From c15abc6fa655f0e5c2a9e3cc3bc7722c7ecc8573 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Thu, 22 Feb 2024 12:22:26 -0500 Subject: [PATCH 06/11] Profile Server module --- src/python/WMCore/REST/Server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/WMCore/REST/Server.py b/src/python/WMCore/REST/Server.py index 1e446cadcc..6c4aa249a9 100644 --- a/src/python/WMCore/REST/Server.py +++ b/src/python/WMCore/REST/Server.py @@ -3,6 +3,7 @@ import cherrypy import inspect +from memory_profiler import profile import os import re import signal @@ -760,6 +761,7 @@ def default(self, *args, **kwargs): default._cp_config = {'response.stream': True} + @profile def _call(self, param): """The real HTTP request handler. From c0e4a9fc6ebfe3e72169526b96a3cbddfef8307c Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Thu, 22 Feb 2024 12:36:49 -0500 Subject: [PATCH 07/11] Profile Format module --- src/python/WMCore/REST/Format.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/python/WMCore/REST/Format.py b/src/python/WMCore/REST/Format.py index 7eae6b1286..cd8e8e36a2 100644 --- a/src/python/WMCore/REST/Format.py +++ b/src/python/WMCore/REST/Format.py @@ -6,6 +6,7 @@ from Utils.PythonVersion import PY3 from Utils.Utilities import encodeUnicodeToBytes, encodeUnicodeToBytesConditional from future.utils import viewitems +from memory_profiler import profile import hashlib import json @@ -566,6 +567,7 @@ def _etag_tail(head, tail, etag): if etagval: cherrypy.response.headers["ETag"] = etagval +@profile def stream_maybe_etag(size_limit, etag, reply): """Maybe generate ETag header for the response, and handle If-Match and If-None-Match request headers. Consumes the reply until at most From 56bdfe48dd909017419d435f6bbb9ad203a4dd11 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Thu, 22 Feb 2024 21:26:58 -0500 Subject: [PATCH 08/11] Temporarily remove etag mechanism in Format module --- src/python/WMCore/REST/Format.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/python/WMCore/REST/Format.py b/src/python/WMCore/REST/Format.py index cd8e8e36a2..311921a7c3 100644 --- a/src/python/WMCore/REST/Format.py +++ b/src/python/WMCore/REST/Format.py @@ -6,7 +6,6 @@ from Utils.PythonVersion import PY3 from Utils.Utilities import encodeUnicodeToBytes, encodeUnicodeToBytesConditional from future.utils import viewitems -from memory_profiler import profile import hashlib import json @@ -567,7 +566,6 @@ def _etag_tail(head, tail, etag): if etagval: cherrypy.response.headers["ETag"] = etagval -@profile def stream_maybe_etag(size_limit, etag, reply): """Maybe generate ETag header for the response, and handle If-Match and If-None-Match request headers. Consumes the reply until at most @@ -608,12 +606,13 @@ def stream_maybe_etag(size_limit, etag, reply): # clients including browsers will ignore them. size = 0 result = [] - for chunk in reply: - result.append(chunk) - size += len(chunk) - if size > size_limit: - res.headers['Trailer'] = 'X-REST-Status' - return _etag_tail(result, reply, etag) + ### FIXME TODO: this block apparently leaks memory + #for chunk in reply: + # result.append(chunk) + # size += len(chunk) + # if size > size_limit: + # res.headers['Trailer'] = 'X-REST-Status' + # return _etag_tail(result, reply, etag) # We've buffered the entire response, but it may be an error reply. The # generator code does not know if it's allowed to raise exceptions, so From 3a0b9adfbac770b7744696fe2022d4edefd1adb2 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 23 Feb 2024 07:23:47 -0500 Subject: [PATCH 09/11] profile etag_tail in Format module --- src/python/WMCore/REST/Format.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/python/WMCore/REST/Format.py b/src/python/WMCore/REST/Format.py index 311921a7c3..2e7967c8ef 100644 --- a/src/python/WMCore/REST/Format.py +++ b/src/python/WMCore/REST/Format.py @@ -2,6 +2,7 @@ import gzip from builtins import str, bytes, object +from memory_profiler import profile from Utils.PythonVersion import PY3 from Utils.Utilities import encodeUnicodeToBytes, encodeUnicodeToBytesConditional @@ -552,6 +553,7 @@ def _etag_match(status, etagval, match, nomatch): if nomatch and ("*" in nomatch or etagval in nomatch): raise cherrypy.HTTPRedirect([], 304) +@profile def _etag_tail(head, tail, etag): """Generator which first returns anything in `head`, then `tail`. Sets ETag header at the end to value of `etag` if it's defined and @@ -607,12 +609,12 @@ def stream_maybe_etag(size_limit, etag, reply): size = 0 result = [] ### FIXME TODO: this block apparently leaks memory - #for chunk in reply: - # result.append(chunk) - # size += len(chunk) - # if size > size_limit: - # res.headers['Trailer'] = 'X-REST-Status' - # return _etag_tail(result, reply, etag) + for chunk in reply: + result.append(chunk) + size += len(chunk) + if size > size_limit: + res.headers['Trailer'] = 'X-REST-Status' + return _etag_tail(result, reply, etag) # We've buffered the entire response, but it may be an error reply. The # generator code does not know if it's allowed to raise exceptions, so From e7f0c0db7c488d4a44f5d9a73204775d317b08b5 Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Fri, 23 Feb 2024 16:40:14 -0500 Subject: [PATCH 10/11] more profiling --- src/python/WMCore/REST/Format.py | 1 + src/python/WMCore/REST/Server.py | 1 + src/python/WMCore/Services/WMStats/WMStatsReader.py | 3 +++ src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py | 3 ++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/python/WMCore/REST/Format.py b/src/python/WMCore/REST/Format.py index 2e7967c8ef..4c2e8554a9 100644 --- a/src/python/WMCore/REST/Format.py +++ b/src/python/WMCore/REST/Format.py @@ -568,6 +568,7 @@ def _etag_tail(head, tail, etag): if etagval: cherrypy.response.headers["ETag"] = etagval +@profile def stream_maybe_etag(size_limit, etag, reply): """Maybe generate ETag header for the response, and handle If-Match and If-None-Match request headers. Consumes the reply until at most diff --git a/src/python/WMCore/REST/Server.py b/src/python/WMCore/REST/Server.py index 6c4aa249a9..10afc6bf22 100644 --- a/src/python/WMCore/REST/Server.py +++ b/src/python/WMCore/REST/Server.py @@ -729,6 +729,7 @@ def metrics(self): return encodeUnicodeToBytes(metrics) @expose + @profile def default(self, *args, **kwargs): """The HTTP request handler. diff --git a/src/python/WMCore/Services/WMStats/WMStatsReader.py b/src/python/WMCore/Services/WMStats/WMStatsReader.py index 9128966235..5d25575ddf 100644 --- a/src/python/WMCore/Services/WMStats/WMStatsReader.py +++ b/src/python/WMCore/Services/WMStats/WMStatsReader.py @@ -4,6 +4,8 @@ from future.utils import viewitems import logging +from memory_profiler import profile + from Utils.IteratorTools import nestedDictUpdate, grouper from WMCore.Database.CMSCouch import CouchServer from WMCore.Lexicon import splitCouchServiceURL, sanitizeURL @@ -304,6 +306,7 @@ def getT0ActiveData(self, jobInfoFlag=False): return self.getRequestByStatus(T0_ACTIVE_STATUS, jobInfoFlag) + @profile def getRequestByStatus(self, statusList, jobInfoFlag=False, limit=None, skip=None, legacyFormat=False): diff --git a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py index c0a3fc9829..19bf868a7d 100644 --- a/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py +++ b/src/python/WMCore/WMStats/CherryPyThreads/DataCacheUpdate.py @@ -33,7 +33,8 @@ def gatherActiveDataStats(self, config): self.logger.info("Getting active data with job info for statuses: %s", WMSTATS_JOB_INFO) jobData = wmstatsDB.getActiveData(WMSTATS_JOB_INFO, jobInfoFlag=self.getJobInfo) self.logger.info("Getting active data with NO job info for statuses: %s", WMSTATS_NO_JOB_INFO) - jobData.update(wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False)) + tempData = wmstatsDB.getActiveData(WMSTATS_NO_JOB_INFO, jobInfoFlag=False) + jobData.update(tempData) self.logger.info("Running setlatestJobData...") DataCache.setlatestJobData(jobData) self.logger.info("DataCache is up-to-date with %d requests data", len(jobData)) From 1cd071ee7280ee591707cb5a66e500304923c01b Mon Sep 17 00:00:00 2001 From: Alan Malta Rodrigues Date: Mon, 26 Feb 2024 19:26:18 -0500 Subject: [PATCH 11/11] more places leaking memory --- src/python/WMCore/REST/Format.py | 2 ++ src/python/WMCore/REST/Server.py | 2 ++ src/python/WMCore/Services/WMStats/WMStatsReader.py | 2 ++ src/python/WMCore/WMStats/DataStructs/DataCache.py | 3 +++ 4 files changed, 9 insertions(+) diff --git a/src/python/WMCore/REST/Format.py b/src/python/WMCore/REST/Format.py index 4c2e8554a9..9fc42b2100 100644 --- a/src/python/WMCore/REST/Format.py +++ b/src/python/WMCore/REST/Format.py @@ -592,6 +592,8 @@ def stream_maybe_etag(size_limit, etag, reply): req = cherrypy.request res = cherrypy.response match = [str(x) for x in (req.headers.elements('If-Match') or [])] + # FIXME TODO this apparently increases wmstatsserver memory + # footprint by half giga byte nomatch = [str(x) for x in (req.headers.elements('If-None-Match') or [])] # If ETag is already set, match conditions and output without buffering. diff --git a/src/python/WMCore/REST/Server.py b/src/python/WMCore/REST/Server.py index 10afc6bf22..1cfa1a0598 100644 --- a/src/python/WMCore/REST/Server.py +++ b/src/python/WMCore/REST/Server.py @@ -862,6 +862,8 @@ def _call(self, param): # Format the response. response.headers['X-REST-Status'] = 100 response.headers['Content-Type'] = format + # FIXME TODO this apparently increases wmstatsserver memory + # footprint by half giga byte etagger = apiobj.get('etagger', None) or SHA1ETag() reply = stream_compress(fmthandler(obj, etagger), apiobj.get('compression', self.compression), diff --git a/src/python/WMCore/Services/WMStats/WMStatsReader.py b/src/python/WMCore/Services/WMStats/WMStatsReader.py index 5d25575ddf..fda2be556a 100644 --- a/src/python/WMCore/Services/WMStats/WMStatsReader.py +++ b/src/python/WMCore/Services/WMStats/WMStatsReader.py @@ -95,6 +95,7 @@ def setDefaultStaleOptions(self, options): options.update(self.defaultStale) return options + @profile def getLatestJobInfoByRequests(self, requestNames): jobInfoByRequestAndAgent = {} @@ -103,6 +104,7 @@ def getLatestJobInfoByRequests(self, requestNames): jobInfoByRequestAndAgent = self._getLatestJobInfo(requestAndAgentKey) return jobInfoByRequestAndAgent + @profile def _updateRequestInfoWithJobInfo(self, requestInfo): if requestInfo: jobInfoByRequestAndAgent = self.getLatestJobInfoByRequests(list(requestInfo)) diff --git a/src/python/WMCore/WMStats/DataStructs/DataCache.py b/src/python/WMCore/WMStats/DataStructs/DataCache.py index a8c238d249..5bb2ee351c 100644 --- a/src/python/WMCore/WMStats/DataStructs/DataCache.py +++ b/src/python/WMCore/WMStats/DataStructs/DataCache.py @@ -2,6 +2,8 @@ from future.utils import viewitems import time + +from Utils.Utilities import getSize from WMCore.ReqMgr.DataStructs.Request import RequestInfo, protectedLFNs class DataCache(object): @@ -22,6 +24,7 @@ def setDuration(sec): @staticmethod def getlatestJobData(): if (DataCache._lastedActiveDataFromAgent): + print(f"Size of DataCache: {getSize(DataCache._lastedActiveDataFromAgent)}") return DataCache._lastedActiveDataFromAgent["data"] else: return {}