f

eve/workers/pod.yml

@@ -34,6 +34,8 @@ spec:
         value: '10000000'
       - name: ENABLE_SENSISION
         value: 't'
+      - name: warpscript.extension.logEvent
+        value: io.warp10.script.ext.logging.LoggingWarpScriptExtension
     resources:
       requests:
         cpu: 300m

warpscript/utapi/createSnapshot.mc2

@@ -1,3 +1,4 @@
+
 {
   'name' 'utapi/createSnapshot'
   'desc'
@@ -40,15 +41,54 @@
     'utapi.checkpoint' 'checkpoint_class' STORE
     'utapi.repair.correction' 'correction_class' STORE
 
+    // will read the key `max_snapshot_size@utapi/createSnapshot` from warp 10 config
+    'max_snapshot_size' 'P6H' MACROCONFIGDEFAULT DURATION 'max_snapshot_size' STORE
+
     // Fetch latest master snapshot
     $read_token $master_snapshot_class $filterLabels $endTimestamp @utapi/fetchFirstRecordBefore
 
+
     FIRSTTICK
     // If we found a snapshot, increment its timestamp so we start at the tick immediatly after
     <% DUP 0 > %>
     <%  1 + %> IFT
     'masterSnapshotTimestamp' STORE // Grab our ending timestamp from the last master snapshot (0 if no snapshots)
 
+    // If snapshot is longer than 6 hours change endTimestamp to be masterSnapshotTimestamp + 6 hours
+    <% $endTimestamp $masterSnapshotTimestamp - $max_snapshot_size > %>
+    <%
+      // If there are no preceding snapshots
+      <% $masterSnapshotTimestamp 0 == %>
+      <%
+        // Find the oldest checkpoint
+        // 'No previous snapshots found, detecting start of checkpoints.' LOGMSG
+        $auth_info ->JSON 
+        { 'class' $checkpoint_class 'labels' $filterLabels } ->JSON
+        @utapi/findOldestRecord 'oldestTimestamp' STORE
+        // If no checkpoints are found 
+        <% $oldestTimestamp -1 == %>
+        <%
+          // 'No checkpoints found. Looking for corrections' LOGMSG
+          $auth_info ->JSON 
+          { 'class' $correction_class 'labels' $filterLabels } ->JSON
+          @utapi/findOldestRecord 'oldestTimestamp' STORE
+          <% $oldestTimestamp -1 == %>
+          <%
+            // 'No corrections or checkpoints found. Nothing to snapshot' LOGMSG
+            STOP
+          %> IFT
+        %>  IFT
+        <% $oldestTimestamp $max_snapshot_size + $endTimestamp < %>
+        <%
+          $oldestTimestamp $max_snapshot_size + 'endTimestamp' STORE
+        %> IFT
+      %> 
+      <%
+        $masterSnapshotTimestamp $max_snapshot_size + 'endTimestamp' STORE 
+      %> IFTE
+      // 'Oversize snapshot detected using ' $endTimestamp TOSTRING + ' as end' + LOGMSG
+    %> IFT
+
     [
       "node"
       ".app"
@@ -91,6 +131,8 @@
         $endTimestamp
         @utapi/fetchFirstRecordBefore
         VALUES 0 GET @utapi/decodeRecord 'record' STORE
+        <% $record 'ops' GET ISNULL %>
+        <% $record {} 'ops' PUT DROP %> IFT
         $fieldSnapshots $record $value PUT DROP
       %> FOREACH
       $previousSnapshots $fieldSnapshots $field PUT DROP
@@ -163,9 +205,6 @@
       $labelSnapshots $created $labelValue PUT DROP
     %> FOREACH
 
-    // $createdSnapshots ->JSON LOGMSG
-
-    // $endTimestamp TOSTRING LOGMSG
     // Fetch all checkpoints since last master snapshot til passed time
     {
       'token' $read_token
@@ -236,7 +275,6 @@
       $labelSnapshots $created $labelValue PUT DROP
     %> FOREACH
 
-    // $createdSnapshots ->JSON LOGMSG
     0 'snapshots' STORE
     // For each of our indexed fields
     $createdSnapshots KEYLIST