Sample scriptsΒΆ
Upload script (Client-to-Server)ΒΆ
import sys, getopt
from threading import local
from time import sleep
from jetstream import JetStream
from jetstream import LocalServer
from test_connection import Location
def main(argv):
remoteServer = '199.195.144.6'
remotePort = 8886
remoteUser = ''
remotePassword = ''
localUser = ''
localPassword = ''
remotePath = ''
localPath = ''
try:
opts, args = getopt.getopt(argv, "hr:p:s:d:u:w:U:W:",
["remote-host=", "remote-port=", "src-path=", "dst-path=", "remote-user=",
"remote-password=", "local-user=", "local-password="])
# print(f'opts: {opts} args: {args}')
except getopt.GetoptError as e:
print(
f'exception - upload_sample.py -r <remote-host> -p <remote-port> -u <remote-user> -U <local-user> -w <remote-password> -W <local-password> -s <src-path> -d <dst-path> Error: {e}')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(
f'./upload_sample.py -r <remote-host> -p <remote-port> -u <remote-user> -U <local-user> -w <remote-password> -W <local-password> -s <src-path> -d <dst-path>')
sys.exit()
elif opt in ("-r", "--remote-host"):
remoteServer = arg
elif opt in ("-p", "--remote-port"):
remotePort = int(arg)
elif opt in ("-s", "--src-path"):
localPath = arg
elif opt in ("-d", "--dst-path"):
remotePath = arg
elif opt in ("-u", "--remote-user"):
remoteUser = arg
elif opt in ("-w", "--remote-password"):
remotePassword = arg
elif opt in ("-U", "--local-user"):
localUser = arg
elif opt in ("-W", "--local-password"):
localPassword = arg
print(f'Remote Server Info: host : {remoteServer}, port : {remotePort}')
print(f'Mapping Info Info: src : {localPath}, dst : {remotePath}')
print(f'Remote Authentication: user : {remoteUser}, password : {remotePassword}')
print(f'Host Authentication: user : {localUser}, password : {localPassword}')
print('Start...')
run(remoteServer, remotePort, remoteUser, remotePassword, localUser, localPassword, localPath, remotePath)
print('Done...')
def run(remoteServer, remotePort, remoteUser, remotePassword, localUser, localPassword, localPath, remotePath):
print(f'Remote Authentication: user : {remoteUser}, password : {remotePassword}')
print(f'Host Authentication: user : {localUser}, password : {localPassword}')
localServer = LocalServer(localUser, localPassword)
local_api = localServer.api()
# print('I am here')
if local_api:
dest = local_api.send.createDestination(destinationAddress=remoteServer, destinationPort=remotePort)
if dest:
mapping = {remotePath: localPath}
manifest = local_api.send.createManifest(mappings=mapping)
print('start create transfer...')
try:
transfer = local_api.send.createTransfer(dest, manifest, userName=remoteUser, password=remotePassword)
print('Transfer has been created...')
except BaseException as err:
print('failed to create transfer...')
print(f"Unexpected err = {err}, type(err) = {type(err)}")
transfer = None
if transfer != None:
while transfer['status'] != 'complete':
print(f"Transfer status: {transfer['status']}")
if transfer['status'] == 'error':
print(f"Error Message : {transfer['errorMessage']}")
break
percent = 0
if transfer['totalBlocks'] == 0:
percent = 100
else:
percent = transfer['blocksAcked'] / transfer['totalBlocks']
print(f'Transfer Progress: {percent * 100}% done')
sleep(5)
transfer = local_api.send.getTransfer(transfer)
else:
percent = transfer['blocksAcked'] / transfer['totalBlocks']
print(f'Transfer Progress: {percent * 100}% done')
if __name__ == "__main__":
print(f'Number of arguments: {len(sys.argv)} arguments.')
print(f'Argument List: {str(sys.argv)}')
main(sys.argv[1:])
Download script (Server-to-Client)ΒΆ
import sys, getopt
from threading import local
from time import sleep
from jetstream import JetStream
from jetstream import LocalServer
from test_connection import TestConnection
from test_connection import Location
def main(argv):
serverAddress = '199.195.144.6'
serverPort = 8886
userName = ''
password = ''
hostUser = ''
hostPassword = ''
try:
opts, args = getopt.getopt(argv,"hr:p:s:d:u:w:U:W:",["remote-host=", "remote-port=","src-path=", "dst-path=", "remote-user=", "remote-password=", "local-user=", "local-password="])
print(f'opts: {opts} args: {args}')
# opts, args = getopt.getopt(argv,"hi:o:",["ifile=","ofile="])
except getopt.GetoptError as e:
print(f'exception - start_transfer.py -r <remote-host> -p <remote-port> -s <src-path> -d <dst-path> Error: {e}')
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('start_transfer.py -r <remote-host> -p <port> -s <source-path> -d <destination-path>')
sys.exit()
elif opt in ("-r", "--remote-host"):
serverAddress = arg
elif opt in ("-p", "--remote-port"):
serverPort = int(arg)
elif opt in ("-s", "--src-path"):
src_path = arg
elif opt in ("-d", "--dst-path"):
dst_path = arg
elif opt in ("-u", "--remote-user"):
userName = arg
elif opt in ("-w", "--remote-password"):
password = arg
elif opt in ("-U", "--local-user"):
hostUser = arg
elif opt in ("-W", "--local-password"):
hostPassword = arg
print(f'Remote Server Info: host : {serverAddress}, port : {serverPort}')
print(f'Mapping Info Info: src : {src_path}, dst : {dst_path}')
print(f'Remote Authentication: user : {userName}, password : {password}')
print(f'Host Authentication: user : {hostUser}, password : {hostPassword}')
print('Start...')
run(serverAddress, serverPort, userName, password, hostUser, hostPassword, src_path, dst_path)
print('Done...')
def connectToServer(serverAddress='localhost', port=8886, userName=None, password=None):
print('start connecting...')
api = JetStream().api()
api.connect(serverAddress, port)
api.server.auth(userName=userName, password=password)
print('done...')
return api
def createDestination(api, destinationAddress, destinationPort):
return api.send.createDestination(destinationAddress=destinationAddress, destinationPort=destinationPort)
def run(serverAddress, serverPort, userName, password, hostUser, hostPassword, src_path, dst_path):
print(f'Remote Authentication: user : {userName}, password : {password}')
print(f'Host Authentication: user : {hostUser}, password : {hostPassword}')
location = Location(serverAddress, serverPort, userName, password)
tc = TestConnection()
tc.setLocation(location)
api = tc.api()
#print('I am here')
if api:
tc.testConnection(api)
localServer = LocalServer()
localServer.setUserName(hostUser)
localServer.setPassword(hostPassword)
local_api = localServer.api()
try:
local_api.connect('localhost', 54321)
local_api.server.auth(hostUser, hostPassword)
except:
print('Failed in local server')
destinationAddress, destinationPort, discoveryResult = tc.testAddressDiscovery(local_api=local_api)
print(discoveryResult)
dest = createDestination(api, destinationAddress=destinationAddress, destinationPort=destinationPort)
if dest:
# Discovery reply for download test.
if local_api:
# Collect the 'senderPort', used in the discover packet to punch a hole though any firewall.
recvSenderPort = dest['senderPort']
# dst_id = dest['destinationId']
ret = local_api.server.sendDiscoverPacket(discoveryResult['discoverId'], location.serverAddress(), recvSenderPort)
print(ret['discoverPacketSent'])
if ret['discoverPacketSent']:
mapping = {dst_path : src_path}
manifest = api.send.createManifest(mappings=mapping)
#destUser = 'jetstream'
#destPassword = 'optimus@12345'
print('start create transfer...')
try:
transfer = api.send.createTransfer(dest, manifest, userName=hostUser, password=hostPassword)
print('done...')
except BaseException as err:
print('failed to create transfer...')
print(f"Unexpected err = {err}, type(err) = {type(err)}")
transfer = None
if transfer != None:
while transfer['status'] != 'complete':
print(f"Transfer status: {transfer['status']}")
if transfer['status'] == 'error':
print(f"Error Message : {transfer['errorMessage']}")
break
percent = 0
if transfer['totalBlocks'] == 0:
percent = 100
else:
percent = transfer['blocksAcked'] / transfer['totalBlocks']
print(f'Transfer Progress: {percent * 100}% done')
sleep(5)
transfer = api.send.getTransfer(transfer)
else:
percent = transfer['blocksAcked'] / transfer['totalBlocks']
print(f'Transfer Progress: {percent * 100}% done')
if __name__ == "__main__":
print(f'Number of arguments: {len(sys.argv)} arguments.')
print(f'Argument List: {str(sys.argv)}')
main(sys.argv[1:])
One-way sync (Server-to-Server)ΒΆ
import sys
sys.path.append('/usr/local/jetstream/share/jetstream-python')
import jetstream
import json
import time
import datetime
wait_time_seconds_resync = 1800 # time interval for next round of sync
file_change_threshold = 30000000 # only files older than this will be transferred; helps avoid moving files being changed
file_time_resolution_usec = 1000000 # timestamp difference needs to be large than this value to be considered "newer"
# uni-directional sync. dst will be forced to be the same as src
def one_way_sync(src_host, src_host_alt, src_port, src_user, src_pass, src_path, dst_host, dst_host_alt, dst_port, dst_user, dst_pass, dst_path):
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s Starting one way sync from %s:%s to %s:%s' % (curTime, src_host, src_path, dst_host, dst_path))
# Connect to both sides.
src = jetstream.Jetstream().api()
src.connect(src_host_alt, src_port)
src.server.auth(src_user, src_pass)
dst = jetstream.Jetstream().api()
dst.connect(dst_host_alt, dst_port)
dst.server.auth(dst_user, dst_pass)
now = int(time.time() * 1000000)
# Get the contents of the directories on both sides.
src_man = src.send.waitForManifest(src.send.createManifest(mappings={dst_path: src_path}))
dst_man = dst.send.waitForManifest(dst.send.createManifest(mappings={src_path: dst_path}))
src_files = src.send.getManifestFiles(src_man)
dst_files = dst.send.getManifestFiles(dst_man)
# Manifest file for an empty dir will contain the empty dir path. Consider these manifests empty instead.
if len(src_files['fileMappings']) == 1:
elem = src_files['fileMappings'][0]
if elem['srcPath'] == src_path and elem['destPath'] == dst_path and elem['isEmptyDir']:
src_files['fileMappings'] = {}
if len(dst_files['fileMappings']) == 1:
elem = dst_files['fileMappings'][0]
if elem['srcPath'] == dst_path and elem['destPath'] == src_path and elem['isEmptyDir']:
dst_files['fileMappings'] = {}
# Make a lookup of src file names.
src_mapping = { e['srcPath']:e for e in src_files['fileMappings'] } # make a dict so we can lookup files in source
# Make a lookup of dst file names.
dst_mapping = { e['srcPath']:e for e in dst_files['fileMappings'] } # make a dict so we can lookup files in dest
for dst_data in dst_files['fileMappings']:
src_file = dst_data['destPath']
dest_file = dst_data['srcPath']
if not src_file in src_mapping:
dst.server.rm(dest_file)
print(' Deleted file %s' % dest_file)
# Make a new manifest that contains newer files in source.
mappings={}
for src_data in src_files['fileMappings']:
if now - src_data['modificationTime'] > file_change_threshold: # check if the file is old enough
dest_file = src_data['destPath']
try:
dst_data = dst_mapping[dest_file]
if not src_data['isEmptyDir'] or not dst_data['isEmptyDir']: # ignore empty dirs on both sides
if src_data['modificationTime'] > (dst_data['modificationTime'] + file_time_resolution_usec): # src is newer
mappings[dest_file] = src_data
except KeyError:
# File doesn't exist in dest. Copy it over.
mappings[dest_file] = src_data
# If there's something to sync, the sync it.
if len(mappings) > 0:
man = src.send.waitForManifest(src.send.createManifest(mappings=mappings))
# Clean up since we don't need these manifests anymore.
src.send.deleteManifest(src_man)
dst.send.deleteManifest(dst_man)
print(' Syncing files:')
for f in mappings.values():
print(' %s (%s)' % (f['srcPath'], f['fileSize']))
# Send over the files.
destination = src.send.createDestination(dst_host, destinationPort=dst_port)
transfer = src.send.createTransfer(destination, man, userName=dst_user, password=dst_pass,
transferFlags=jetstream.constants.TRANSFER_FLAG_READ_WRITE |
jetstream.constants.TRANSFER_FLAG_WRITE_MTIME |
jetstream.constants.TRANSFER_FLAG_WRITE_ONLY_NEWER,
fileTimeResolutionMicroseconds=file_time_resolution_usec)
transfer = src.send.waitForTransfer(transfer)
if transfer['status'] == 'complete':
print('Done.')
else:
print(' Error. Status: %s: %s' % (transfer['status'], transfer['errorMessage']))
else:
print(' Nothing to sync')
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s One way sync done' % (curTime))
def run():
syncOps = []
# sync op2 - fromDSBX
op1 = {}
op1['src_host'] = '10.0.0.1'
op1['src_host_local'] = 'localhost'
op1['src_use_local'] = True
op1['src_port'] = 8886
op1['src_user'] = r'user1'
op1['src_pass'] = r'password1'
op1['src_path'] = r'/Path/to/folder_1'
op1['dst_host'] = '188.255.182.74'
op1['dst_host_local'] = '188.255.182.74'
op1['dst_use_local'] = False
op1['dst_port'] = 8886
op1['dst_user'] = r'user2'
op1['dst_pass'] = r'password2'
op1['dst_path'] = r'/Path/to/folder_2'
syncOps.append(op1)
wait_time_seconds = 3 # time between sync operations
while True:
for syncOp in syncOps:
src_host_loc = syncOp['src_host']
if syncOp['src_use_local']:
src_host_loc = syncOp['src_host_local']
dst_host_loc = syncOp['dst_host']
if syncOp['dst_use_local']:
dst_host_loc = syncOp['dst_host_local']
try:
one_way_sync(syncOp['src_host'], src_host_loc, syncOp['src_port'], syncOp['src_user'], syncOp['src_pass'], syncOp['src_path'], syncOp['dst_host'], dst_host_loc, syncOp['dst_port'], syncOp['dst_user'], syncOp['dst_pass'], syncOp['dst_path'])
except Exception as e: # pylint: disable=broad-except
print(e)
time.sleep(wait_time_seconds)
time.sleep(wait_time_seconds_resync)
run()
Bi-directional sync (Server-to-Server)ΒΆ
import sys
sys.path.append('/usr/local/jetstream/share/jetstream-python')
import jetstream
import json
import time
import datetime
wait_time_seconds_resync = 1800 # time interval for next round of sync
file_change_threshold = 30000000 # only files older than this will be transferred; helps avoid moving files being changed
file_time_resolution_usec = 1000000 # timestamp difference needs to be large than this value to be considered "newer"
# bi-directional sync with two records of src and dest snapshots, must be run twice (second call with src and dst inverted). file trim applies to both src and dst.
def sync(src_host, src_host_alt, src_port, src_user, src_pass, src_path, dst_host, dst_host_alt, dst_port, dst_user, dst_pass, dst_path, src_persistent, dst_persistent):
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s Starting sync from %s:%s to %s:%s' % (curTime, src_host, src_path, dst_host, dst_path))
# Connect to both sides.
src = jetstream.Jetstream().api()
src.connect(src_host_alt, src_port)
src.server.auth(src_user, src_pass)
dst = jetstream.Jetstream().api()
dst.connect(dst_host_alt, dst_port)
dst.server.auth(dst_user, dst_pass)
now = int(time.time() * 1000000)
# Get the contents of the directories on both sides.
src_man = src.send.waitForManifest(src.send.createManifest(mappings={dst_path: src_path}))
dst_man = dst.send.waitForManifest(dst.send.createManifest(mappings={src_path: dst_path}))
src_files = src.send.getManifestFiles(src_man)
dst_files = dst.send.getManifestFiles(dst_man)
# Manifest file for an empty dir will contain the empty dir path. Consider these manifests empty instead.
if len(src_files['fileMappings']) == 1:
elem = src_files['fileMappings'][0]
if elem['srcPath'] == src_path and elem['destPath'] == dst_path and elem['isEmptyDir']:
src_files['fileMappings'] = {}
if len(dst_files['fileMappings']) == 1:
elem = dst_files['fileMappings'][0]
if elem['srcPath'] == dst_path and elem['destPath'] == src_path and elem['isEmptyDir']:
dst_files['fileMappings'] = {}
# Make a lookup of dst file names.
dst_mapping = { e['srcPath']:e for e in dst_files['fileMappings'] } # make a dict so we can lookup files in dest
# To handle src deletes, we compare the last known state of the source files compared to the current state.
# If files are missing and don't have a newer version in dst, then delete.
try:
with open(src_persistent) as file:
old_src_files = json.load(file)
except Exception: # pylint: disable=broad-except
old_src_files={}
if 'fileMappings' in old_src_files and len(old_src_files['fileMappings']) > 0:
src_mapping = { e['srcPath']:e for e in src_files['fileMappings'] } # make a dict so we can lookup files in source
for old_src_data in old_src_files['fileMappings']:
src_file = old_src_data['srcPath']
dest_file = old_src_data['destPath']
try:
if not src_file in src_mapping: # no longer exists in src
dst_data = dst_mapping[dest_file]
if dst_data['modificationTime'] <= (old_src_data['modificationTime'] + file_time_resolution_usec): # dst exists and isn't newer
dst.server.rm(dest_file)
print(' Deleted file %s' % dest_file)
elif dst_data['isEmptyDir'] and old_src_data['isEmptyDir']: # clean up empty dirs
dst.server.rm(dest_file)
print(' Deleted empty dir %s' % dest_file)
except KeyError:
pass
# To handle dst deletes, load the last known state of the dest files. Don't copy over src if dest used to have
# that file but it was deleted (unless the src version is newer than the deleted dest version).
try:
with open(dst_persistent) as file:
old_dst_files = json.load(file)
except Exception: # pylint: disable=broad-except
old_dst_files={}
if 'fileMappings' in old_dst_files and len(old_dst_files['fileMappings']) > 0:
old_dst_mapping = { e['srcPath']:e for e in old_dst_files['fileMappings'] } # make a dict so we can lookup old files in dest
else:
old_dst_mapping = {}
# Make a new manifest that contains newer files in source.
mappings={}
for src_data in src_files['fileMappings']:
if now - src_data['modificationTime'] > file_change_threshold: # check if the file is old enough
dest_file = src_data['destPath']
try:
dst_data = dst_mapping[dest_file]
if not src_data['isEmptyDir'] or not dst_data['isEmptyDir']: # ignore empty dirs on both sides
if src_data['modificationTime'] > (dst_data['modificationTime'] + file_time_resolution_usec): # src is newer
mappings[dest_file] = src_data
except KeyError:
# File doesn't exist in dest. Check if it was already deleted.
try:
old_dst_data = old_dst_mapping[dest_file]
if not src_data['isEmptyDir'] or not old_dst_data['isEmptyDir']: # ignore empty dirs on both sides
if src_data['modificationTime'] > (old_dst_data['modificationTime'] + file_time_resolution_usec): # src is newer than deleted file
mappings[dest_file] = src_data
except KeyError:
mappings[dest_file] = src_data
# If there's something to sync, the sync it.
syncOk = True
if len(mappings) > 0:
syncOk = False
man = src.send.waitForManifest(src.send.createManifest(mappings=mappings))
# Clean up since we don't need these manifests anymore.
src.send.deleteManifest(src_man)
dst.send.deleteManifest(dst_man)
print(' Syncing files:')
for f in mappings.values():
print(' %s (%s)' % (f['srcPath'], f['fileSize']))
# Send over the files.
destination = src.send.createDestination(dst_host, destinationPort=dst_port)
transfer = src.send.createTransfer(destination, man, userName=dst_user, password=dst_pass,
transferFlags=jetstream.constants.TRANSFER_FLAG_READ_WRITE |
jetstream.constants.TRANSFER_FLAG_WRITE_MTIME |
jetstream.constants.TRANSFER_FLAG_WRITE_ONLY_NEWER,
fileTimeResolutionMicroseconds=file_time_resolution_usec)
transfer = src.send.waitForTransfer(transfer)
if transfer['status'] == 'complete':
syncOk = True
print('Done.')
else:
print(' Error. Status: %s: %s' % (transfer['status'], transfer['errorMessage']))
else:
print(' Nothing to sync')
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s Sync done' % (curTime))
# Write out the last successful source and destination mappings.
if syncOk:
with open(src_persistent, 'w') as file:
json.dump(src_files, file)
def run():
syncOps = []
# sync op1 - share
op1 = {}
op1['src_host'] = '10.0.0.1'
op1['src_host_local'] = 'localhost' # used when this script is running on the src server host and domain name cannot be resolved locally
op1['src_use_local'] = True
op1['src_port'] = 8886
op1['src_user'] = r'user1'
op1['src_pass'] = r'password1'
op1['src_path'] = r'/Path/to/folder_1'
op1['src_persistent'] = r'/Users/jetadmin/jetstream_sync'
op1['dst_host'] = '188.255.182.74'
op1['dst_host_local'] = '188.255.182.74'
op1['dst_use_local'] = False
op1['dst_port'] = 8886
op1['dst_user'] = r'user2'
op1['dst_pass'] = r'password2'
op1['dst_path'] = r'/Path/to/folder_2'
op1['dst_persistent'] = r'/Users/jetadmin/jetstream_sync'
op1['bidirectional'] = True
syncOps.append(op1)
wait_time_seconds = 3 # time between sync operations
while True:
for syncOp in syncOps:
src_host_loc = syncOp['src_host']
time.sleep(wait_time_seconds)
if syncOp['bidirectional']:
try:
sync(syncOp['dst_host'], dst_host_loc, syncOp['dst_port'], syncOp['dst_user'], syncOp['dst_pass'], syncOp['dst_path'], syncOp['src_host'], src_host_loc, syncOp['src_port'], syncOp['src_user'], syncOp['src_pass'], syncOp['src_path'], syncOp['dst_persistent'], syncOp['src_persistent'])
except Exception as e: # pylint: disable=broad-except
print(e)
time.sleep(wait_time_seconds)
time.sleep(wait_time_seconds_resync)
run()
Uni-directional sync (Server-to-Server)ΒΆ
import sys
sys.path.append('/usr/local/jetstream/share/jetstream-python')
import jetstream
import json
import time
import datetime
# uni-directional or bi-directional sync (if run second time with src dst inverted). file trim will not happen. missing file on one side will be recovered from the other side.
def non_trim_sync(src_host, src_host_atl, src_port, src_user, src_pass, src_path, dst_host, dst_host_alt, dst_port, dst_user, dst_pass, dst_path):
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s Starting non-trim sync from %s:%s to %s:%s' % (curTime, src_host, src_path, dst_host, dst_path))
# Connect to both sides.
src = jetstream.Jetstream().api()
src.connect(src_host_atl, src_port)
src.server.auth(src_user, src_pass)
dst = jetstream.Jetstream().api()
dst.connect(dst_host_alt, dst_port)
dst.server.auth(dst_user, dst_pass)
now = int(time.time() * 1000000)
# Get the contents of the directories on both sides.
src_man = src.send.waitForManifest(src.send.createManifest(mappings={dst_path: src_path}))
dst_man = dst.send.waitForManifest(dst.send.createManifest(mappings={src_path: dst_path}))
src_files = src.send.getManifestFiles(src_man)
dst_files = dst.send.getManifestFiles(dst_man)
# Manifest file for an empty dir will contain the empty dir path. Consider these manifests empty instead.
if len(src_files['fileMappings']) == 1:
elem = src_files['fileMappings'][0]
if elem['srcPath'] == src_path and elem['destPath'] == dst_path and elem['isEmptyDir']:
src_files['fileMappings'] = {}
if len(dst_files['fileMappings']) == 1:
elem = dst_files['fileMappings'][0]
if elem['srcPath'] == dst_path and elem['destPath'] == src_path and elem['isEmptyDir']:
dst_files['fileMappings'] = {}
# Make a lookup of dst file names.
dst_mapping = { e['srcPath']:e for e in dst_files['fileMappings'] } # make a dict so we can lookup files in dest
# Make a new manifest that contains newer files in source.
mappings={}
for src_data in src_files['fileMappings']:
if now - src_data['modificationTime'] > file_change_threshold: # check if the file is old enough
dest_file = src_data['destPath']
try:
dst_data = dst_mapping[dest_file]
if not src_data['isEmptyDir'] or not dst_data['isEmptyDir']: # ignore empty dirs on both sides
if src_data['modificationTime'] > (dst_data['modificationTime'] + file_time_resolution_usec): # src is newer
mappings[dest_file] = src_data
except KeyError:
# File doesn't exist in dest.
mappings[dest_file] = src_data
# If there's something to sync, the sync it.
if len(mappings) > 0:
man = src.send.waitForManifest(src.send.createManifest(mappings=mappings))
# Clean up since we don't need these manifests anymore.
src.send.deleteManifest(src_man)
dst.send.deleteManifest(dst_man)
print(' Syncing files:')
for f in mappings.values():
print(' %s (%s)' % (f['srcPath'], f['fileSize']))
# Send over the files.
destination = src.send.createDestination(dst_host, destinationPort=dst_port)
transfer = src.send.createTransfer(destination, man, userName=dst_user, password=dst_pass,
transferFlags=jetstream.constants.TRANSFER_FLAG_READ_WRITE |
jetstream.constants.TRANSFER_FLAG_WRITE_MTIME |
jetstream.constants.TRANSFER_FLAG_WRITE_ONLY_NEWER,
fileTimeResolutionMicroseconds=file_time_resolution_usec)
transfer = src.send.waitForTransfer(transfer)
if transfer['status'] == 'complete':
print(' Done.')
else:
print(' Error. Status: %s: %s' % (transfer['status'], transfer['errorMessage']))
else:
print(' Nothing to sync')
curTime = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('%s Non-trim sync done' % (curTime))
def run():
syncOps = []
# sync op2 - fromDSBX
op1 = {}
op1['src_host'] = '10.0.0.1'
op1['src_host_local'] = 'localhost'
op1['src_use_local'] = True
op1['src_port'] = 8886
op1['src_user'] = r'user1'
op1['src_pass'] = r'password1'
op1['src_path'] = r'/Path/to/folder_1'
op1['dst_host'] = '188.255.182.74'
op1['dst_host_local'] = '188.255.182.74'
op1['dst_use_local'] = False
op1['dst_port'] = 8886
op1['dst_user'] = r'user2'
op1['dst_pass'] = r'password2'
op1['dst_path'] = r'/Path/to/folder_2'
syncOps.append(op1)
wait_time_seconds = 3 # time between sync operations
while True:
for syncOp in syncOps:
src_host_loc = syncOp['src_host']
if syncOp['src_use_local']:
src_host_loc = syncOp['src_host_local']
dst_host_loc = syncOp['dst_host']
if syncOp['dst_use_local']:
dst_host_loc = syncOp['dst_host_local']
if syncOp['non_trim_sync']:
try:
non_trim_sync(syncOp['dst_host'], dst_host_loc, syncOp['dst_port'], syncOp['dst_user'], syncOp['dst_pass'], syncOp['dst_path'], syncOp['src_host'], src_host_loc, syncOp['src_port'], syncOp['src_user'], syncOp['src_pass'], syncOp['src_path'])
except Exception as e: # pylint: disable=broad-except
print(e)
time.sleep(wait_time_seconds)
time.sleep(wait_time_seconds_resync)
run()