feat: add low-memory-mode option for automagic

.github/workflows/integration-test.yml

@@ -10,7 +10,7 @@ jobs:
       - name: setup Nim
         uses: jiro4989/setup-nim-action@v1
         with:
-          nim-version: '2.x' # default is 'stable'
+          nim-version: '2.0.4' # default is 'stable'
 
       - name: clone takajo
         uses: actions/checkout@v4

.github/workflows/nim.yml

@@ -29,7 +29,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: jiro4989/setup-nim-action@v1
         with:
-          nim-version: '2.x' # default is 'stable'
+          nim-version: '2.0.4' # default is 'stable'
       - name: Build check
         run: nimble build --threads:on -Y
       - name: Run tests

.github/workflows/release.yml

@@ -20,6 +20,8 @@ jobs:
 
       - name: Setup Nim
         uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: '2.0.4'
 
       - name: Update nimble libraries
         run: |

src/takajo.nim

@@ -111,6 +111,7 @@ when isMainModule:
             help = {
                 "displayTable": "display the results table (default: false)",
                 "level": "specify the minimum alert level (default: informational)",
+                "low-memory-mode": "Scan with minimal memory usage by processing each command sequentially (default: false)",
                 "output": "output directory (default: case-1)",
                 "quiet": "do not display the launch banner (default: false)",
                 "skipProgressBar": "do not display the progress bar (default: false)",

src/takajopkg/automagic.nim

@@ -6,7 +6,7 @@ type
 
 proc autoMagic(level: string = "informational", skipProgressBar: bool = false,
     displayTable: bool = false, output: string = "case-1", quiet: bool = false,
-    timeline: string) =
+    timeline: string, lowMemoryMode:bool = false) =
   checkArgs(quiet, timeline, level)
   if dirExists(output):
     echo "The directory '" & output & "' exists. Please specify a new folder name."
@@ -147,4 +147,7 @@ proc autoMagic(level: string = "informational", skipProgressBar: bool = false,
   let cmd = AutoMagicCmd(level: level, skipProgressBar: skipProgressBar,
     displayTable: displayTable, output: output, timeline: timeline,
     name: "automagic", msg: AutoMagicMsg)
-  cmd.analyzeJSONLFile(cmds)
+  if lowMemoryMode:
+    cmd.analyzeJSONLFileLowMemory(cmds)
+  else:
+    cmd.analyzeJSONLFile(cmds)

src/takajopkg/takajoCore.nim

@@ -82,3 +82,47 @@ proc analyzeJSONLFile*(self: AbstractCmd, cmds: seq[AbstractCmd] = newSeq[
     table.echoTableSepsWithStyled(seps = boxSeps)
 
   outputElapsedTime(startTime)
+
+
+proc analyzeJSONLFileLowMemory*(self: AbstractCmd, cmds: seq[AbstractCmd] = newSeq[
+    AbstractCmd]()) =
+  let startTime = epochTime()
+  var commands = cmds
+  var bar: SuruBar
+  if not self.skipProgressBar:
+    bar = initSuruBar()
+    bar[0].total = countJsonlAndStartMsg(self.name, self.msg, self.timeline) * cmds.len()
+    bar.setup()
+  var resultSummaryTable: seq[CmdResult] = @[]
+  let fileInfo = getFileInfo(self.timeline)
+  var filePaths = @[self.timeline]
+  if (fileInfo.kind == pcDir or fileInfo.kind == pcLinkToDir):
+    filePaths = getTargetExtFileLists(self.timeline, ".jsonl", true)
+  for cmd in commands:
+    for path in filePaths:
+      for line in lines(path):
+        if not self.skipProgressBar:
+          inc bar
+          bar.update(1000000000)
+        let jsonLineOpt = parseLine(line)
+        if jsonLineOpt.isNone:
+          continue
+        let jsonLine: HayabusaJson = jsonLineOpt.get()
+        try:
+          if cmd.filter(jsonLine):
+            cmd.analyze(jsonLine)
+        except CatchableError:
+          continue
+    cmd.resultOutput()
+    resultSummaryTable.add(cmd.cmdResult)
+  if not self.skipProgressBar:
+    bar.finish()
+
+  var table: TerminalTable
+  table.add @["Command", "Results", "Saved Files"]
+  for i, result in enumerate(resultSummaryTable):
+    table.add commands[i].name, result.results, result.savedFiles
+  echo ""
+  table.echoTableSepsWithStyled(seps = boxSeps)
+
+  outputElapsedTime(startTime)