Skip to content

Commit

Permalink
Add Snapshotter
Browse files Browse the repository at this point in the history
Closes: #2
  • Loading branch information
vdusek committed Feb 15, 2024
1 parent 314ac93 commit 19dbc42
Show file tree
Hide file tree
Showing 6 changed files with 341 additions and 19 deletions.
12 changes: 12 additions & 0 deletions src/crawlee/_utils/math.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,3 +25,15 @@ def weighted_avg(values: list[float], weights: list[float]) -> float:
raise ValueError('Total weight cannot be zero')

return result / total_weight


def to_mb(bytes_: int) -> int:
"""Converts bytes to megabytes (MB).
Args:
bytes_: The number of bytes to convert.
Returns:
int: The equivalent size in megabytes.
"""
return round(bytes_ / (1024**2))
19 changes: 19 additions & 0 deletions src/crawlee/autoscaling/memory_info.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Inspiration: https://github.com/apify/crawlee/blob/master/packages/utils/src/internals/memory-info.ts

from dataclasses import dataclass


@dataclass
class MemoryInfo:
"""Describes memory usage of the process."""

total_bytes: int # Total memory available in the system or container
free_bytes: int # Amount of free memory in the system or container
used_bytes: int # Amount of memory used (= totalBytes - freeBytes)
main_process_bytes: int # Amount of memory used by the current Python process
child_processes_bytes: int # Amount of memory used by child processes of the current Python process


async def get_memory_info() -> MemoryInfo:
# TODO
return MemoryInfo(1, 2, 3, 4, 5)
Loading

0 comments on commit 19dbc42

Please sign in to comment.