Skip to content

Commit

Permalink
Merge pull request #55 from HPCNow/main
Browse files Browse the repository at this point in the history 
Final development commit before extensive test development. froster v0.12.0
  • Loading branch information
@victormachadoperez
victormachadoperez authored Jun 5, 2024
2 parents b256a18 + ed2e700 commit ef6ba12
Show file tree
Hide file tree
Showing 4 changed files with 84 additions and 72 deletions.
6 changes: 3 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,21 @@ Froster is a user-friendly archiving tool for teams that move data between highe

```
sudo apt-get update
sudo apt-get install -y curl pipx git gcc lib32gcc-s1 unzip fuse3
sudo apt-get install -y curl pipx gcc lib32gcc-s1 unzip fuse3
```

### On RHEL

```
sudo yum update
sudo yum install -y curl pipx git gcc lib32gcc-s1 unzip fuse3 python3-devel
sudo yum install -y curl pipx gcc lib32gcc-s1 unzip fuse3 python3-devel
```

### On HPC machine

Please contact your administrator to install these packages:
```
curl pipx git gcc lib32gcc-s1 unzip fuse3 python3.xx-devel
curl pipx gcc lib32gcc-s1 unzip fuse3 python3.xx-devel
```

</br>
Expand Down
136 changes: 80 additions & 56 deletions froster/froster.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ def __init__(self):
# Last timestamp we checked for an updated
self.update_check_timestamp = config.get(
'UPDATE', 'timestamp', fallback=None)

# Shared configuration
self.is_shared = config.getboolean(
'SHARED', 'is_shared', fallback=False)
Expand Down Expand Up @@ -1210,6 +1210,9 @@ def check_update(self):

if hasattr(self, 'update_check_timestamp') and self.update_check_timestamp is not None:
# Check if last day was less than 86400 * 7 = (1 day) * 7 = 1 week
print(timestamp)
print(self.update_check_timestamp)
print(timestamp - self.update_check_timestamp)
if timestamp - self.update_check_timestamp < (86400*7):
# Less than a week since last check
return False
Expand Down Expand Up @@ -3241,7 +3244,7 @@ def _index_locally(self, folder):
except Exception:
print_error()

def _slurm_cmd(self, folders, cmd_type):
def _slurm_cmd(self, folders, cmd_type, scheduled):
'''Execute the current command using SLURM'''

try:
Expand All @@ -3262,7 +3265,8 @@ def _slurm_cmd(self, folders, cmd_type):
se.submit_job(cmd=cmd,
cmd_type=cmd_type,
label=label,
shortlabel=shortlabel)
shortlabel=shortlabel,
scheduled=scheduled)

except Exception:
print_error()
Expand Down Expand Up @@ -4458,41 +4462,50 @@ def restore(self, folders, aws: AWSBoto):
f' froster archive --permissions "/your/folder/to/archive"\n', file=sys.stderr)
return

if use_slurm(self.args.noslurm):
self._slurm_cmd(folders=folders, cmd_type='restore')

else:
# Archive locally all folders. If recursive flag set, archive all subfolders too.
for folder in folders:
for root, dirs, files in self._walker(folder):
# Archive locally all folders. If recursive flag set, archive all subfolders too.
for folder in folders:
for root, dirs, files in self._walker(folder):

# Break in case of non-recursive restore
if not is_recursive and root != folder:
break
# Break in case of non-recursive restore
if not is_recursive and root != folder:
break

archived_folder_info = self.froster_archives_get_entry(
root)
archived_folder_info = self.froster_archives_get_entry(
root)

if archived_folder_info is None:
print(f'\nFolder {root} is not archived')
print(f'No entry found in froster-archives.json\n')
continue
if archived_folder_info is None:
print(f'\nFolder {root} is not archived')
print(f'No entry found in froster-archives.json\n')
continue

if not self._contains_non_froster_files(root):
print(f'\nWARNING: Folder {root} contains non-froster metadata files')
print('Has this folder been deleted using "froster delete" command?.')
print('Please empty the folder before restoring.\n')
continue
if not self._contains_non_froster_files(root):
print(
f'\nWARNING: Folder {root} contains non-froster metadata files')
print(
'Has this folder been deleted using "froster delete" command?.')
print('Please empty the folder before restoring.\n')
continue

if self._restore_locally(root, aws):
if self._restore_locally(root, aws):
# Already restored

# If nodownload flag is set we are done
if self.args.nodownload:
print(
f'\nFolder restored but not downloaded (--no-download flag set)\n')
return
else:
# If nodownload flag is set we are done
if self.args.nodownload:
print(
f'\nFolder restored but not downloaded (--no-download flag set)\n')
return
else:
self._download(root)
else:
# Restore ongoing
# In this case the slurm will only be used for downloading. AWS has taken care of the restore
if is_slurm_installed() and not self.args.noslurm:
# schedule execution in 12 hours
self._slurm_cmd(
folders=folders, cmd_type='restore', scheduled=12)



except Exception:
print_error()
Expand All @@ -4518,7 +4531,8 @@ def _restore_verify(self, source, target):
return

# Get the path to the hashfile
hashfile = os.path.join(restpath, self.md5sum_restored_filename)
hashfile = os.path.join(
restpath, self.md5sum_restored_filename)

# Create the Rclone object
rclone = Rclone(self.args, self.cfg)
Expand All @@ -4539,7 +4553,8 @@ def _restore_verify(self, source, target):
os.remove(tar_path)
print(' ...done\n')

where_did_file_go_full_path = os.path.join(target, self.where_did_the_files_go_filename)
where_did_file_go_full_path = os.path.join(
target, self.where_did_the_files_go_filename)
if os.path.exists(where_did_file_go_full_path):
os.remove(where_did_file_go_full_path)

Expand Down Expand Up @@ -5581,7 +5596,7 @@ def _reorder_sbatch_lines(self, script_buffer):
except Exception:
print_error()

def submit_job(self, cmd, cmd_type, label, shortlabel):
def submit_job(self, cmd, cmd_type, label, shortlabel, scheduled=None):
'''Submit a Slurm job'''

try:
Expand All @@ -5594,6 +5609,9 @@ def submit_job(self, cmd, cmd_type, label, shortlabel):
f'#SBATCH --job-name=froster:{cmd_type}:{shortlabel}')
self.add_line(f'#SBATCH --cpus-per-task={self.args.cores}')
self.add_line(f'#SBATCH --mem={self.args.memory}')
if scheduled:
self.add_line(f'#SBATCH --begin={scheduled}')
self.add_line(f'#SBATCH --requeue')
self.add_line(f'#SBATCH --output={output_dir}-%J.out')
self.add_line(f'#SBATCH --mail-type=FAIL,REQUEUE,END')
self.add_line(f'#SBATCH --mail-user={self.cfg.email}')
Expand Down Expand Up @@ -6201,7 +6219,8 @@ def subcmd_archive(self, arch: Archiver, aws: AWSBoto):
return

if not aws.check_credentials():
print('\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
print(
'\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
sys.exit(1)

# Check if the user provided the hotspots argument
Expand Down Expand Up @@ -6231,7 +6250,8 @@ def subcmd_restore(self, arch: Archiver, aws: AWSBoto):

try:
if not aws.check_credentials():
print('\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
print(
'\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
sys.exit(1)

if self.args.monitor:
Expand Down Expand Up @@ -6274,9 +6294,10 @@ def subcmd_delete(self, arch: Archiver, aws: AWSBoto):

try:
if not aws.check_credentials():
print('\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
print(
'\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
sys.exit(1)

if not self.args.folders:

# Get the list of folders from the archive
Expand Down Expand Up @@ -6311,9 +6332,10 @@ def subcmd_mount(self, arch: Archiver, aws: AWSBoto):

try:
if not aws.check_credentials():
print('\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
print(
'\nError: invalid credentials. Check the AWS configuration with "froster config --aws"\n')
sys.exit(1)

if self.args.list:
arch.print_current_mounts()
return
Expand Down Expand Up @@ -6454,19 +6476,20 @@ def subcmd_credentials(self, cfg: ConfigManager, aws: AWSBoto):
print(' ...AWS credentials are NOT valid\n')
return False

def subcmd_update(self):
def subcmd_update(self, mute_no_update):
'''Check if an update is available'''
try:

cmd = "curl -s https://api.github.com/repos/dirkpetersen/froster/releases"
cmd = "curl -s https://api.github.com/repos/hpcnow/froster/releases"

result = subprocess.run(cmd, shell=True, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)

if result.returncode != 0:
print(f"Error checking if froster update available. Command run: {cmd}: {result.stderr.strip()}")
print(
f"Error checking if froster update available. Command run: {cmd}: {result.stderr.strip()}")
return False

def compare_versions(version1, version2):
v1 = [int(v) for v in version1.split(".")]
v2 = [int(v) for v in version2.split(".")]
Expand All @@ -6477,7 +6500,7 @@ def compare_versions(version1, version2):
if v1_part != v2_part:
return v1_part - v2_part
return 0

releases = json.loads(result.stdout)
if not releases:
print('Note: Could not check for updates')
Expand All @@ -6488,10 +6511,12 @@ def compare_versions(version1, version2):

if compare_versions(latest, current) > 0:
print(f'\nA froster update is available: froster v{latest}')
print(f'You can update froster using the command:')
print(f' curl -s https://raw.githubusercontent.com/dirkpetersen/froster/main/install.sh?$(date +%s) | bash\n')
print(f'\nYou can update froster using the command:')
print(
f' curl -s https://raw.githubusercontent.com/dirkpetersen/froster/main/install.sh?$(date +%s) | bash\n')
else:
print(f'\nFroster is up to date: froster v{current}\n')
if not mute_no_update:
print(f'\nFroster is up to date: froster v{current}\n')

except Exception:
print_error()
Expand Down Expand Up @@ -6710,19 +6735,18 @@ def parse_arguments(self):
parser_restore.add_argument('folders', action='store', default=[], nargs='*',
help='folders you would like to to restore (separated by space)')


parser_restore.add_argument('-a', '--aws', dest='aws', action='store_true',
help="Restore folder on new AWS EC2 instance instead of local machine")

parser_restore.add_argument('-d', '--days', dest='days', action='store', default=30,
help='Number of days to keep data in S3 One Zone-IA storage at $10/TiB/month (default: 30)')

parser_restore.add_argument('-i', '--instance-type', dest='instancetype', action='store', default="",
help='The EC2 instance type is auto-selected, but you can pick any other type here')

parser_restore.add_argument('-l', '--no-download', dest='nodownload', action='store_true',
help="skip download to local storage after retrieval from Glacier")

parser_restore.add_argument('-m', '--monitor', dest='monitor', action='store_true',
help="Monitor EC2 server for cost and idle time.")

Expand Down Expand Up @@ -6945,13 +6969,13 @@ def main():
elif args.subcmd in ['credentials', 'crd']:
cmd.subcmd_credentials(cfg, aws)
elif args.subcmd in ['update', 'upd']:
cmd.subcmd_update()
cmd.subcmd_update(mute_no_update=False)
else:
cmd.print_help()

# Check if there are updates on froster every X days
if cfg.check_update():
cmd.subcmd_update()
if cfg.check_update() and args.subcmd not in ['update', 'upd']:
cmd.subcmd_update(mute_no_update=True)

# Close the AWS session
aws.close_session()
Expand Down
12 changes: 0 additions & 12 deletions install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,18 +129,6 @@ check_apt_dependencies() {
exit 1
fi

# Check if git is installed (pipx installation requirement)
# TODO: Get rid of this requirement once froster is in PyPi repository
if [[ -z $(command -v git) ]]; then
echo "Error: git is not installed."
echo
echo "Please install git"
echo "In most linux distros you can install the latest version of git by running the following commands:"
echo " sudo apt update"
echo " sudo apt install -y git"
echo
exit 1
fi

# Check if unzip is installed (rclone requirement)
if [[ -z $(command -v unzip) ]]; then
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "froster"
version = "0.11.8"
version = "0.12.0"
description = "Froster is a tool for easy data transfer between local file systems and AWS S3 storage."
authors = ["Victor Machado <[email protected]>"]
readme = "README.md"
Expand Down

0 comments on commit ef6ba12

Please sign in to comment.