Examples#
Examples will be extended in future.
Introduction#
Most slk and slk_helpers commands are available as functions pyslk.pyslk.slk_COMMAND(...). All command lines arguments of slk version 3.3.10 are available as the functions’ arguments. These functions are simple wrappers that print out the text, which the slk/slk_helpers commands normally print to the command line. A bit more advanced wrappers are available for a few commands via pyslk.parsers.slk_COMMAND_....
Basic examples#
Print Version#
>>> import pyslk
>>> pyslk.cli_versions()
{'slk': 'SCLI Version 3.3.90', 'slk_helpers': 'Version 1.9.7'}
List files and directories in specific path#
# load library
>>> import pyslk
# list content of a folder
>>> pyslk.list("/arch/bm0146/k204221/iow")
permissions owner group filesize date time filename
0 -rwxr-xr-x- k204221 bm0146 1.268945e+06 2020-06-10 08:25 /arch/bm0146/k204221/iow/INDEX.txt
1 -rw-r--r--t k204221 bm0146 2.094216e+10 2020-06-05 17:36 /arch/bm0146/k204221/iow/iow_data2_001.tar
2 -rw-r--r--t k204221 bm0146 2.034971e+10 2020-06-05 17:38 /arch/bm0146/k204221/iow/iow_data2_002.tar
3 -rw-r--r--t k204221 bm0146 2.088344e+10 2020-06-05 17:38 /arch/bm0146/k204221/iow/iow_data2_003.tar
4 -rw-r--r--t k204221 bm0146 2.071567e+10 2020-06-05 17:40 /arch/bm0146/k204221/iow/iow_data2_004.tar
5 -rw-r--r--t k204221 bm0146 2.047869e+10 2020-06-05 17:40 /arch/bm0146/k204221/iow/iow_data2_005.tar
6 -rw-r--r--t k204221 bm0146 8.364491e+09 2020-06-05 17:41 /arch/bm0146/k204221/iow/iow_data2_006.tar
7 -rw-r--r--t k204221 bm0146 2.007016e+11 2020-06-05 19:37 /arch/bm0146/k204221/iow/iow_data3_001.tar
8 -rw-r--r--t k204221 bm0146 2.646606e+10 2020-06-05 19:14 /arch/bm0146/k204221/iow/iow_data3_002.tar
9 -rw-r--r--t k204221 bm0146 4.194304e+06 2020-06-05 19:43 /arch/bm0146/k204221/iow/iow_data4_001.tar
10 -rw-r--r--t k204221 bm0146 1.128477e+10 2020-06-05 19:46 /arch/bm0146/k204221/iow/iow_data4_002.tar
11 -rw-r--r--t k204221 bm0146 2.094216e+10 2020-06-10 08:21 /arch/bm0146/k204221/iow/iow_data5_001.tar
12 -rw-r--r--t k204221 bm0146 2.034971e+10 2020-06-10 08:23 /arch/bm0146/k204221/iow/iow_data5_002.tar
13 -rw-r--r--t k204221 bm0146 2.088344e+10 2020-06-10 08:23 /arch/bm0146/k204221/iow/iow_data5_003.tar
14 -rw-r--r--t k204221 bm0146 2.071567e+10 2020-06-10 08:24 /arch/bm0146/k204221/iow/iow_data5_004.tar
15 -rw-r--r--t k204221 bm0146 2.047869e+10 2020-06-10 08:25 /arch/bm0146/k204221/iow/iow_data5_005.tar
16 -rw-r--r--t k204221 bm0146 8.364491e+09 2020-06-10 08:25 /arch/bm0146/k204221/iow/iow_data5_006.tar
17 -rw-r--r--t k204221 bm0146 2.094216e+10 2020-06-05 17:53 /arch/bm0146/k204221/iow/iow_data_001.tar
18 -rw-r--r--t k204221 bm0146 2.034971e+10 2020-06-05 17:53 /arch/bm0146/k204221/iow/iow_data_002.tar
19 -rw-r--r--t k204221 bm0146 2.088344e+10 2020-06-05 17:56 /arch/bm0146/k204221/iow/iow_data_003.tar
20 -rw-r--r--- k204221 bm0146 2.071567e+10 2020-06-05 17:56 /arch/bm0146/k204221/iow/iow_data_004.tar
21 -rw-r--r--- k204221 bm0146 2.047869e+10 2020-06-05 17:58 /arch/bm0146/k204221/iow/iow_data_005.tar
22 -rw-r-----t k204221 bm0146 8.364491e+09 2020-06-05 17:57 /arch/bm0146/k204221/iow/iow_data_006.tar
Check if files exist#
# load library
>>> import pyslk
# check if a file exists
>>> pyslk.resource_exists("/arch/bm0146/k204221/iow/iow_data2_001.tar")
True
# check another non-existing file
>>> pyslk.resource_exists("/arch/bm0146/k204221/iow/NOT_EXISTS.tar")
False
Print metadata of a file#
# load library
>>> import pyslk
# get metadata of a file
>>> pyslk.get_metadata("/arch/bm0146/k204221/iow/INDEX.txt")
{'netcdf.Title': 'ABC DEF GHI'}
Get size of a file#
# load library
>>> import pyslk
# get size
>>> pyslk.get_resource_size("/arch/bm0146/k204221/iow/INDEX.txt")
1268945
Get size of one folder recursively#
# load library
>>> import pyslk
# get size
>>> pyslk.arch_size("/arch/bm0146/k204221/iow", unit='G')
{'value': 534.3052, 'unit': 'G'}
Check if a file is in the cache and retrieve it#
# load library
>>> import pyslk
>>> import os
# is the file stored in the cache? yet!
>>> pyslk.is_cached("/arch/bm0146/k204221/iow/INDEX.txt")
True
# retrieve file
>>> pyslk.retrieve("/arch/bm0146/k204221/iow/INDEX.txt", '.', preserve_path=True)
''
>>> os.path.exists('INDEX.txt')
True
Get the checksums of a file#
# load library
>>> import pyslk
# get checksums
>>> pyslk.checksum("/arch/bm0146/k204221/iow/INDEX.txt")
{'adler32': '412dec3a', 'sha512': '83644ffc384c70aaf60d29f3303d771e219f3b3f8abdb3a590ac501faf7abbeb371ee4b6b949a1cdc3fa7c44438cd6643f877e8c1ab27d8fe5cc26eb06896d99'}
Search for files based on certain conditions#
We want to search for files, which are stored in /arch/bm0146/k204221/iow, are larger than 1 kb and smaller
than approximately 2 MB.
# load pyslk
>>> import pyslk
# generate search query
>>> search_query = pyslk.gen_search_query(["path=/arch/bm0146/k204221/iow", "resources.size>1024", "resources.size<2000000"])
>>> search_query
'{"$and":[{"resources.size":{"$gt":1024}},{"resources.size":{"$lt":2000000}},{"path":{"$gte":"/arch/bm0146/k204221/iow","$max_iterations":1}}]}'
# run search
>>> search_id = pyslk.search(search_query)
>>> search_id
321321
# list search results
>>> pyslk.list(search_id)
permissions owner group filesize date time filename
0 -rwxr-xr-x- k204221 bm0146 1268945.0 2020-06-10 08:25 /arch/bm0146/k204221/iow/INDEX.txt
or just
> import pyslk
> pyslk.list(pyslk.search(pyslk.gen_search_query(["path=/arch/bm0146/k204221/iow", "resources.size>1024", "resources.size<2000000"])))
permissions owner group filesize date time filename
0 -rwxr-xr-x- k204221 bm0146 1268945.0 2020-06-10 08:25 /arch/bm0146/k204221/iow/INDEX.txt
Optimize retrieval of several files by running one retrieval per tape#
>>> import pprint
>>> import pyslk
# create grouping
>>> grouping = pyslk.group_files_by_tape('/arch/bm0146/k204221/iow', recursive=True, run_search_query=True)
# look into the grouping
>>> pprint.pprint(grouping)
[{'barcode': '',
'description': 'files currently stored in the HSM cache',
'file_count': 3,
'file_ids': [49058705518, 49058705517, 49058705497],
'files': ['/arch/bm0146/k204221/iow/iow_data_005.tar',
'/arch/bm0146/k204221/iow/iow_data_004.tar',
'/arch/bm0146/k204221/iow/INDEX.txt'],
'id': -1,
'location': 'cache',
'search_complete': True,
'search_id': 505589,
'search_query': '{"$and":[{"path":{"$gte":"/arch/bm0146/k204221/iow"}},{"smart_pool":"slpstor"}]}',
'status': ''},
...
{'barcode': 'M12217M8',
'description': 'files stored on the tape with the given ID and barcode/label',
'file_count': 2,
'file_ids': [49058705503, 49058705502],
'files': ['/arch/bm0146/k204221/iow/iow_data2_006.tar',
'/arch/bm0146/k204221/iow/iow_data2_005.tar'],
'id': 130879,
'location': 'tape',
'search_complete': True,
'search_id': 505599,
'search_query': '{"$and":[{"path":{"$gte":"/arch/bm0146/k204221/iow"}},{"tape_barcode":"M12217M8"}]}',
'status': 'AVAILABLE'}]
# print the search ids
>>> [r['search_id'] for r in grouping]
[505589, 505590, 505591, 505592, 505593, 505594, 505595, 505596, 505597, 505598, 505599]
# run retrievals
>>> pyslk.retrieve(505589, '.', preserve_path=True)
>>> pyslk.retrieve(505590, '.', preserve_path=True)
>>> pyslk.retrieve(505591, '.', preserve_path=True)
>>> pyslk.retrieve(505592, '.', preserve_path=True)
>>> pyslk.retrieve(505593, '.', preserve_path=True)
>>> pyslk.retrieve(505594, '.', preserve_path=True)
>>> pyslk.retrieve(505595, '.', preserve_path=True)
>>> pyslk.retrieve(505596, '.', preserve_path=True)
>>> pyslk.retrieve(505597, '.', preserve_path=True)
>>> pyslk.retrieve(505598, '.', preserve_path=True)
>>> pyslk.retrieve(505599, '.', preserve_path=True)