#!/bin/ksh -p

# For more detailed documentation, see zrep.txt or zrep.overview.txt
ZREP_VERSION=1.0.1

######## start of included files from zrep_top here
########### zrep_vars

# This should basically be included as common vars before all zrep stuff.
# It contains all 'constant' definitions, as well as a few crucial
# shared routines, such as lock handling ones.

# These first three, are user tunables
SSH=${SSH:-ssh}
ZREP_PATH=${ZREP_PATH:-zrep}  #Set to /path/to/zrep, if needed, for remote
#ZREP_CREATE_FLAGS="-o whatever"   #Set for extra options on remote zfs create

#########################################################################
# Everyting else below here, should not be touched. First we have autodetect
# routines, and then internal utilities such as locking functions.


Z_LOCAL_HOST=`uname -n`
Z_LOCAL_HOST=${Z_LOCAL_HOST%%.*}

# Solaris hack, to use native perl, which isnt always in $PATH,but should
# always be there. It's also simple, straightforward, and non-extended.
# But on other OSs will fall back to use default perl.
PERL_BIN=${PERL_BIN:-/usr/perl5/bin}

# Capability for this  does not show in usage.
# So, just tie this to MU6 related check,like HAS_SNAPPROPS
PROPTYPES="local,received" 

# dump the usage message, and check for capabilities
# make sure we dont spew for non-root, so that "zrep status" works
case `whoami` in
	root)
	zrep_checkfile=/var/run/zrep.check.$$
	;;
	*)
	zrep_checkfile=/tmp/zrep.check.$$
	;;
esac


zfs >$zrep_checkfile 2>&1 
# Sigh. solaris 11 changes output yet again, there is no global usage msg,
# which makes it much less efficient to parse.
# So first 'if' clause is a bit of a hack.
# Just assume has everything, if usage message is newer style

if grep 'help' $zrep_checkfile >/dev/null ;then
	Z_HAS_X=1	  # can use recv -x
	Z_HAS_SNAPPROPS=1 # can set properties on snapshots
			  # This also lets me set accurate "last synced" timestamps
			  # otherwise cant use zrep:sent sanely
			  # would lose it on rollbacks

	DEPTHCAP="-d 1" # limits "list -r"
else
	if grep 'receive.*-x' $zrep_checkfile >/dev/null ;then
		Z_HAS_X=1	  # can use recv -x
	else
		Z_HAS_X=0
	fi
	if grep 'set.*snapshot' $zrep_checkfile >/dev/null ;then
		Z_HAS_SNAPPROPS=1 # can set properties on snapshots
	else
		Z_HAS_SNAPPROPS=0
		PROPTYPES="local"
	fi
	if grep 'list.*-d' $zrep_checkfile >/dev/null ;then
		DEPTHCAP="-d 1" # limits "list -r"
	else
		DEPTHCAP=""
		print WARNING: old ZFS version detected
		print WARNING: You may not nest zrep managed filesystems
	fi
fi

if ((Z_HAS_X)) ; then
	Z_HAS_O=1  # can recv use -o option
else
	Z_HAS_O=0
fi
rm $zrep_checkfile


Z_LOCK_RETRY=${Z_LOCK_RETRY:-10}  # default 10 second  retry, 1 per sec
Z_SAVE_COUNT=${Z_SAVE_COUNT:-5}
Z_GLOBAL_LOCKFILE=/var/run/zrep.lock

if [[ "$Z_GLOBAL_PID" == "" ]] ; then
	export Z_GLOBAL_PID=$$
fi

Z_SETHOLD=${Z_SETHOLD:-"zfs hold"}
# if your zfs isnt new enough, and you like to live dangerously, 
# you can skip setting holds by using this instead.
# Although I may not have gotten around to using this in the code either!
#Z_SETHOLD="echo skipping zfs hold on"




# return 0 if "we" are holding lock, 1 otherwise
# Note that we check for "us, OR our global parent", if different
#
zrep_has_global_lock(){
	[ ! -f "${Z_GLOBAL_LOCKFILE}" ] && return 1
	lockpid=`cat ${Z_GLOBAL_LOCKFILE}`
	if [[ "$lockpid" == "" ]] ; then return 1 ; fi
	if [[ "$lockpid" != "$Z_GLOBAL_PID" ]] ; then
		if [[ "$lockpid" != "$$" ]] ; then
			return 1
		fi
	fi

	return 0
}



#Note: it is an ERROR to call this if you already have lock
#It is binary, not recursive ownership.
zrep_get_global_lock(){
	typeset retry_count=$Z_LOCK_RETRY

	if  [ ! -f "${Z_GLOBAL_LOCKFILE}" ] ; then
            echo $Z_GLOBAL_PID > $Z_GLOBAL_LOCKFILE
            return 0
        fi

	# otherwise, deal with fail
	# Check for dead old holder first.
	# CANNOT CLEAN UP OURSELVES: race condition problems.
	
	while (( retry_count > 0 )); do
		sleep 1
		if  [ ! -f "${Z_GLOBAL_LOCKFILE}" ] ; then
		    echo $Z_GLOBAL_PID > $Z_GLOBAL_LOCKFILE
		    return 0
		fi
		retry_count=$((retry_count-1))
	done

	print Failed to acquire global lock 
	return 1
}

zrep_release_global_lock(){
	if zrep_has_global_lock ; then
		rm $Z_GLOBAL_LOCKFILE
		return $?
	else
		print ERROR: zrep_release_global_lock called, but do not own lock
		return 1
	fi
}

# returns PID of zrep process holding a lock on filesystem, if there is one.
# NOTE: prints "-" NOT "", if lock unheld
zrep_fs_lock_pid(){
	zfs get -H -o value zrep:lock-pid $1
}
zrep_has_fs_lock(){
	typeset check=`zfs get -H -o value zrep:lock-pid $1`
	if ((check == $$)) ; then
	return 0
	else
	return 1
	fi
}

# use global lock first (if not already), then
# grab lock on individual fs
# return 1 on fail, 0 on lock acquired
# Note that it is an ERROR to call this, if you already have lock
# Note2: if a dead process has lock, it will forcibly override and
# acqure lock
zrep_lock_fs(){
	# global lock is slow. so do quickcheck first.
	typeset check=`zrep_fs_lock_pid $1` newcheck
	if [[ "$check" != "-" ]] ; then
		# validate fs lock before giving up
	    if [ "${check}" != "-" ] ; then
                [ -f "${Z_GLOBAL_LOCKFILE}" ] && return 1
            fi
	fi

	zrep_get_global_lock  || return 1

	# Double-check if needed, now that we have global lock
	if [[ "$check" != "-" ]] ; then
		newcheck=`zrep_fs_lock_pid $1`
		if [[ "$newcheck" != "$check" ]] && [[ "$newcheck" != "-" ]]
		then
			# oops. someone else must have dealt with it.
			# If they havent reset it to "-" then give up
			zrep_release_global_lock
			return 1
		fi
		print DEBUG: overiding stale lock on $1 from pid $check >/dev/fd/2
	fi
	
	zfs set zrep:lock-pid=$$ $1
	zfs set zrep:lock-time=`date +%Y%m%d%H%M%S` $1
	zrep_release_global_lock 
}

# release lock, if we have it.
# Since this could be called by an exit cleanup routine blindly,
# dont exit program if we dont have lock. But do return error
zrep_unlock_fs(){
	typeset lockpid=`zrep_fs_lock_pid $1`
	if ((lockpid != $$)) ; then return 1; fi

	#since "we" already have it locked, no need to get global lock first
	zfs inherit zrep:lock-time $1
	zfs inherit zrep:lock-pid $1

	return 0
}


# Quit whole program with error status, outputting args to stderr
# Release global lock if we are holding it
#   Unless we're running in parallel batch mode
#   I'll need to plan that out more carefully!
#
zrep_errquit(){
	print Error: "$@" >/dev/fd/2

	if zrep_has_global_lock ; then
		if [[ "$$" -ne "$Z_GLOBAL_PID" ]] ; then
			print EXTRA-ERROR: Running in child proc.
			print 'Not sure whether to release global lock. NOT releasing!'
			exit 1
		else

			zrep_release_global_lock
		fi
	fi
	exit 1
}

# Optimization wrapper for ssh: if destination host is ourself, dont use ssh.
# Just run the local command mentioned
# Be careful about quotes here. In fact, try not to use any.
# Usage:  zrep_ssh desthost  commands_for_ssh go_here
zrep_ssh(){
	typeset ssh_cmd
	case "$1" in
		localhost|$Z_LOCAL_HOST)
			ssh_cmd=""
			;;
		*)
			ssh_cmd="$SSH $1"
			;;
	esac
	shift
	$ssh_cmd "$@"
}

zrep_gettimeinseconds(){
	# unfortunately, solaris date doesnt do '%s', so need to use perl
	    date +%s
}
###### zrep_status

# be sure to have included zrep_vars

# This file contains all "status" related routines.
# It should be folded into final "zrep" script




#
#Give this a top level zrep registered filesystem, NOT snapshot.
# Will print out various status points, such as last sync date.
# Or if given no args, will print out sync date for all zrep mastered fs
# Note that the date given is time of SNAPSHOT, not time sync completed.
#
zrep_status(){
	typeset check fs srcfs jdesthost destfs date lastsnap verbose=0
	typeset printall=0
	
	if [[ "$1" == "-v" ]] ; then
		verbose=1 ; shift
	fi

	if [[ "$1" == "" ]] ; then
		set -- `zrep_list_master`
	elif [[ "$1" == "-a" ]] ; then
		set -- `zrep_list`
		printall=1
	fi

	while [[ "$1" != "" ]] ; do
		fs="$1"

		destfs=`zfs get -H -o value zrep:dest-fs $fs`
		if [[ "$destfs" == "-" ]] || [[ "$destfs" == "" ]]; then
			zrep_errquit "$fs is not a zrep registered filesystem"
		fi

		lastsnap=`getlastsnapsent $fs`
		if [[ "$lastsnap" == "" ]] ; then
			date="[NEVER]"
		else
			date=`zfs get -H -o value creation $lastsnap`
		fi
		if ((printall)) && ((verbose)) ; then
			# If we are printing out  ALL filesystems,
			# then we have to make sure left side is always
			#  "src filesystem", not "named filesystem"
			# then we have to check what the src fs is
			srcfs=`zfs get -H -o value zrep:src-fs $fs`
		else
			# Yes, okay, if -a is used, then
			# technically, this isnt always "src".
			# but it prints out right, so close enough :)
			srcfs="$fs"
		fi

		if ((verbose)) ; then
			desthost=`zfs get -H -o value zrep:dest-host $srcfs`
			printf "%-25s->%-35s %s\n" $srcfs "$desthost:$destfs" "${date#????}"
		else
			printf "%-47s" $srcfs
			print "last synced $date"
		fi
		shift
	done
}


# convenience function to list only local filesystems for which we are
# zrep master for.
# In contrast, zrep_list, lists ALL zrep registered filesystem, at the moment.
# 
# Annoyingly... it would be way faster if we could just stick with the
# pure "zfs get" implementation, but we also need to deal with the zone
# issue. When a single zfs filesystem is visible aross multiple zones,
# we dont want them all thinking they are master
#
# Durn. Individual validation required.
zrep_list_master(){
	typeset srchost
	for fs in `zfs get -H -o name -s local zrep:master "$@"` ; do
		srchost=`zfs get -H -o value zrep:src-host $fs`
		if [[ "$srchost" == "$Z_LOCAL_HOST" ]] ; then
			print $fs
		fi
	done
}


# Given ONE filesystem, print all zrep properties for it.
# Note that this is internal routine. we do not validate input.
list_verbose(){
	print $1:
	zfs get -H -o property,value -s $PROPTYPES all $1
}


# Note: called by both user, AND by zrep_status
#
# Usage:
# zrep_list [-v]
# zrep_list [-L]
# zrep_list [-v] fs1 fs2
# list all zrep-initialized filesystems (NOT snapshots..)
# If no specific fs listed, will show master, AND received filesystems,
#    unless -L given (in which case, only local masters will be shown)
#
# Normal output is one line per fs.
#
#  -v gives all properties of each filesystem
#  Give only one of -L or -v 
#
# This works because we only set this property on the actual fs.
# "source type" on snapshots for this property is "inherited" not local
#  or "received"
zrep_list(){
	typeset fslist="" verbose=0
	typeset printcmd="zfs get -H -o name -s $PROPTYPES zrep:dest-fs"

	case $1 in
		-v)
		verbose=1
		printcmd=list_verbose
		shift
		;;
		-L)
		printcmd="zrep_list_master"
		shift
		;;
	esac



	if [[ "$1" != "" ]] ; then
		while [[ "$1" != "" ]] ; do
			if zfs list -t filesystem $1 >/dev/null 2>&1 ; then
				$printcmd $1
			else
				zrep_errquit "Expecting filesystem, but got $1"
			fi
			shift
		done
		return
	fi

	# Must be "list all" now. But which output format?

	# if not verbose, we have a nice shortcut
	if (( $verbose == 0))  ; then
		$printcmd
		return
	fi


	# oh well. have to step through them one by one now, to
	# print out the properties associated with each zrep filesystem

	fslist=`zfs get -H -o name -s $PROPTYPES zrep:dest-fs`

	for fs in $fslist ; do
		print ${fs}:
		# sneaky cheat: only user-set properties will
		# match these 'source' types. So "grep zrep:" is not
		# neccessary. Although we may pick up other user-set values,
		# but that is not neccessarily a bad thing
		zfs get -H -o property,value -s $PROPTYPES all $fs
		print ""
	done
	
	
}

################ zrep_snap
# be sure to have included zrep_vars

# This file contains routines related to 
# "make new snapshot, using next sequence number".
# So it thus includes all snap sequence related routines
# It may contain "sync snapshot" related routines for now.
# It also is definitive for the format of snapshot names
# It also contains most "query status of snaps" type routines,
#  such as "getlastsnapsent"
#
# Normal style for making a snapshot and syncing it:
#   1.  create a snapshot.
#   2.  sync it over
#   3.  set "zrep:sent" on *snapshot*, with timestamp in seconds
# Old-nasty-zfs compat mode:
#   Step 3. Add/update "zrep:lastsent->snapname", and 
#           "zrep:lastsenttime->timestamp",  on *filesystem*
#
######################################################################


# By observation, 'zfs list' shows snapshots order of creation.
# last listed, should be last in sequence.
# But, dont take chances!!
getlastsequence(){
	typeset lastval
	#remember, filesystems can have '_' in them
	getlastsnap $1|sed 's/.*@zrep_\(......\).*/\1/'
}

# prints out last snapshot zrep created, going purely by sequence.
# Note: "last created", which may or may NOT be "last successfully synced". 
# This is basically "getallsnaps |tail -1"
getlastsnap(){
	zfs list -t snapshot -H -o name $DEPTHCAP -r $1 |
	   sed -n '/@zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]/'p |
	   sort | tail -1
}

# Usage:   getlastsnapsent zpool/FSNAME
getlastsnapsent(){
	# arg.  more efficient if we can just return value directly,
	# but i'm using backwards compat :(
	typeset lastsent
	lastsent=`zfs get  -H -o name -r -s local zrep:sent $1 |
	   sort | tail -1`
	if [[ "$lastsent" != "" ]] ; then
		print $lastsent
		return
	fi

	# Fallback method, for backwards compat with older ZFS code,
	# since it cant set properties on snapshots
	zfs get  -H -o value -s local zrep:lastsent $1
}

# outputs time in seconds, of when the last successful sync for the
# filesystem was done. (in format compatible with zrep_gettimeinseconds() )
#  Note that this is time of actual sync, not snapshot creation time.
#
# This unfortunately needs to be compatible with both new way, and
# old-nasty-hack-way
# 
# In future, may take optional argument of which HOST to check 
# sync with. But since I currently only suport one host per fs... oh well.
# If never synced, will return 1, and print "-"
#
getlastsynctime(){
	typeset fs lastsent senttime

	if [[ "$1" == "" ]] ; then
		zrep_errquit Internal error: no arg to getlastsynctime
	fi
	fs="$1"

	# Deal with possibly upgraded system;
	#   Check "lastsent", only as fallback.

	# copy from getlastsnapsent, but only using newest method
	lastsent=`zfs get  -H -o name -r -s local zrep:sent $fs |
	   sort | tail -1`
	senttime=`zfs get  -H -o value zrep:sent $lastsent`
	if [[ "$senttime" != "-" ]] ; then print $senttime ; return 0; fi

	# ooops. try fallback to nasty old zfs-compat style
	senttime=`zfs get -H -o value zrep:lastsent $fs`
	print $senttime
	if [[ "$senttime" != "-" ]] ; then return 0; fi
	return 1
}

#This is for synctosnap, and also zrep_expire
getallsnaps(){
	zfs list -t snapshot -H -o name $DEPTHCAP -r $1 |
	   sed -n '/@zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]/'p |
	   sort
}

# list all snapshots of the given filesystem, that are made by this prog
# arg: fs
list_autosnaps(){
	if [[ "$1" == "" ]] ; then
		zrep_errquit "zrep internalerror: no arg for list_autosnaps"
	fi
 	zfs list $DEPTHCAP -r -H -o name -t snapshot $1 |
		grep '@zrep_[0-9a-f][0-9a-f]'
	# Make sure this format matches other routines in here
	# Okay to just check first few digits though
}

# User entrypoint. Part of pair: snaponly, sendonly
# Just makes snapshot. 
zrep_snaponly(){
	typeset srcfs

	while [[ "$1" != "" ]] ; do
		srcfs="$1" ;shift
		zrep_lock_fs $srcfs
		if [[ $? -ne 0 ]] ; then
			# this function is supposed to be coordinated by user
			# therefore, if something else is competing,
			# coordination has failed. no retry.
			zrep_errquit zrep snaponly failed for $srcfs: cannot get lock
			
		fi

		makesnap $srcfs ||zrep_errquit snaponly for $srcfs failed

		zrep_unlock_fs $srcfs
		
	done	
}

# 
# creates next snapshot in sequence
# consider holding lock here
# Caller must have zrep lock on filesystem:
#   we verify with zrep_has_fs_lock
makesnap(){
	typeset  check oldseq newseq="" newseqX newsnap

	#sanity checks first!
	check="`zfs get -H -o value zrep:src-host $1`"
	if [[ "$check" != "$Z_LOCAL_HOST" ]] ; then
		print ERROR: we are not master host for $1 >/dev/fd/2
		print master is $check, we are $Z_LOCAL_HOST  >/dev/fd/2
		exit 1
	fi

	zrep_has_fs_lock $1
	if [[ $? -ne 0 ]] ; then
		print Internal error:  makesnap fail, no lock on $1 >/dev/fd/2
		exit 1
	fi

	oldseq=`getlastsequence $1`
	newseq=$((0x$oldseq))

	newseqX=$(printf "%.6x" $(($newseq + 1))  )


	#print DEBUG old=$oldseq new=$newseqX >/dev/fd/2
	newsnap="$1@zrep_$newseqX"
	
	zfs snapshot $newsnap
	if [[ $? -eq 0 ]] ; then
		print  $newsnap; return 0
	else
		return 1
	fi
	
}

## This is the implentation for the "zrep clear" command
## Purpose is to remove all zrep related hooks from a local filesystem. 
##  (NOT delete it)
## Will remove zrep snapshots and zfs zrep: properties
zrep_clear(){
	print "WARNING: Removing all zrep configs and snapshots from $1"
	print Continuing in 10 seconds
	sleep 10

	print Destroying any zrep-related snapshots from $1
	snaplist=`list_autosnaps $1`
	for snap in $snaplist ; do
		zfs destroy -r $snap
	done

	print Removing zrep-related properties from $1
	proplist=`zfs get -H -o property all $1|grep zrep:`
	for prop in $proplist ; do
		zfs inherit $prop $1
	done
}

## This is a special internal routine, used only by zrep_init.
## call with "srcfs  errmsg1 errmsg2..."
## It will REMOVE REMOTEFS if set in PROPERTIES!!
clearquit(){
	remhost=`zfs get -H -o value zrep:dest-host $1`
	remfs=`zfs get -H -o value zrep:dest-fs $1`
	if [[ $? -eq 0 ]] && [[ "$remhost" != "-" ]] && [[ "$remfs" != "-" ]]; then
		zrep_ssh $remhost zfs destroy -r $remfs
	fi

	zrep_clear $1
	shift
	zrep_errquit "$@"
}


# Shared internal routine.
# Set the to/from properties on a fs for zrep
# Called by zrep_init  and zrep_changeconfig
setfsconfigs(){
	typeset srcfs="$1" desthost="$2" destfs="$3" fsname
	if [[ "$destfs" == "" ]] ; then
		zrep_errquit "zrep: no dest fs specified"
	fi
	zfs list $srcfs >/dev/null ||zrep_errquit "filesystem $srcfs must exist already"

	fsname=${srcfs##*/}
	case $destfs in
		# Originally, I had this passthrough only if fsname was at end
		# However,we must allow destfs to have different leaf name,
		# for circumstances such as replication to same host
		*/*)
			:
			;;
		*)
			# Only the pool name given. Let's make it explicit.
			destfs=$destfs/$fsname
			;;
	esac

	zfs set zrep:src-fs=$srcfs		$srcfs
	zfs set zrep:src-host=$Z_LOCAL_HOST	$srcfs
	zfs set zrep:dest-fs=$destfs		$srcfs
	zfs set zrep:dest-host=$desthost	$srcfs
	zfs set zrep:savecount=$Z_SAVE_COUNT	$srcfs
}


# Follow "initial set up" in workflow.txt
# Some day, will allow init from pre-existing snaps< But not today!
# Note that remote fs must share same stem name as source. (for now?)
zrep_init(){
	typeset srcfs="$1" desthost="$2" destfs="$3" fsname snap check vol=0
	if [[ "$srcfs" == "" ]] ; then
		zrep_errquit "zrep: no fs specified"
	fi

	#sanity checks
	check="`zfs get -H -o value zrep:dest-fs $srcfs`"
	if [[ "$check" != "-" ]] ; then
		print "$srcfs is at least partially configured by zrep"
		zrep_errquit "To re-initialize, first use zrep clear $srcfs"
	fi
	check="`zfs get -H -o value type $srcfs`"
	if [[ "$check" == "volume" ]] ; then
		vol=1
		if ((! Z_HAS_O )) ; then
			print "Sorry, your zfs is too old for zrep to handle volume initialization"
			zrep_errquit "Please initialize volume target by hand, if you won't upgrade"
		fi
	fi
	

	print Setting properties on $srcfs
	setfsconfigs $srcfs $desthost $destfs

	#setfsconfigs may do some "smarts" to adjust value, so get it again.
	destfs=`zfs get -H -o value zrep:dest-fs $srcfs`

	if (( Z_HAS_O )) ; then
		READONLYPROP="-o readonly=on"
	else
		READONLYPROP=""
#		print Ancient local version of ZFS detected.
		print Creating destination filesystem as separate step
		zrep_ssh $desthost zfs create $ZREP_CREATE_FLAGS -o readonly=on $destfs || zrep_errquit "Cannot create $desthost:$destfs"
	fi

	snap="${srcfs}@zrep_000000"
	print Creating snapshot $snap
	zfs snapshot $snap || clearquit $srcfs "Cannot create initial snapshot $snap"

	# Note that we may not want to use -p for normal zrep syncs
	# We also should not use -F for normal recv. See workflow.txt
	# Note: we may have to reset readonly=on, if we used -p on send...
	#
	print Sending initial replication stream to $desthost:$destfs
	if (( $Z_HAS_X )) ; then
		# This is the nice, clean, modern codepath, to send
		# zrep settings over automatically at first init
		#
		# But check to see if sending volume or filesystem first,
		# and act appropriately
		#
		typeset MOUNTFILTER
		if (( vol )) ; then
			MOUNTFILTER=""
		else
			MOUNTFILTER="-x mountpoint"
		fi

		zfs send -p $snap |
		  zrep_ssh $desthost zfs recv $MOUNTFILTER $READONLYPROP -F $destfs
	else
		## arg.. Patch your systems!!
		# Doesn't support "recv -x mountpoint", so cant use -p in send
		# This means we have to manually set props lower down as well.
		zfs send $snap |
		  zrep_ssh $desthost zfs recv $READONLYPROP -F $destfs
	fi
	if [[ $? -ne 0 ]] ; then
		clearquit $srcfs "Error transferring $snap to $desthost:$destfs. Resetting"
	fi

	# Successful initial sync! Woo! okay record that, etc.
	# ... after stupid old-zfs-compat junk, that is
	if (( ! Z_HAS_X )) ; then
#		print Debug: Because you have old zfs support, setting remote properties by hand
		zrep_ssh $desthost zfs set readonly=on $destfs	||
			clearquit Could not set readonly for $desthost:$destfs

		zrep_ssh $desthost zfs set zrep:src-fs=$srcfs $destfs
		zrep_ssh $desthost zfs set zrep:src-host=$Z_LOCAL_HOST $destfs
		zrep_ssh $desthost zfs set zrep:dest-fs=$destfs $destfs
		zrep_ssh $desthost zfs set zrep:dest-host=$destfs $destfs
		zrep_ssh $desthost zfs set zrep:savecount=$Z_SAVE_COUNT	$destfs

	fi

	# Make sure to set format to match what zrep_sync() looks for!
	if (( Z_HAS_SNAPPROPS )) ; then
		typeset sentprop="zrep:sent=`zrep_gettimeinseconds`"
		zfs set $sentprop ${snap}
	else
		# Arg stupidold stuff cant set props on a snapshot
		# So we have to manually set these on both sides also,
		# "Just in case"
		zfs set zrep:lastsent=${snap} $srcfs
		zrep_ssh $desthost zfs set zrep:lastsent=${snap} $destfs
	fi

	# make sure the above ' set 's (sent, lastsent)
	# match what zrep_sync() does !!!


	# Note: we have to set master property NOW, not before,
	# because "recv -x zrep:master" Does Not Work properly
	# Also, it avoids things like "zrep sync all" from attempting
	# to sync it before initial sync has been done.
	# We don't even have to zrep_lock_fs until this is set
	
	zfs set zrep:master=yes	$srcfs

	print Initialization copy of $srcfs to $desthost:$destfs complete

}


zrep_changeconfig(){
	typeset srcfs="$1" desthost="$2" destfs="$3" check
	if [[ "$srcfs" == "" ]] ; then
		zrep_errquit "zrep: no fs specified"
	fi

	check=`getlastsnap $srcfs`
	if [[ "$check" == "" ]] ; then
		print "No pre-existing zrep snapshots found on $srcfs" >/dev/fd/2
		print $srcfs is not initialized for zrep. cannot change config. >/dev/fd/2
		zrep_errquit Use zrep init on $srcfs instead
	fi

	setfsconfigs $srcfs $desthost $destfs
	
}



##### zrep_sync

# contains meat of the "sync" level operations.
# basic snap routines, and init rountines, are in zrep_snap


####################
# synctosnap: called by zrep_sync, if a specific snapshot is specified.
#
# This LOCAL side, *and*  REMOTE side, match up with local zrep_created 
# snapshot. ...
# 
# Note that it uses zrep_lock_fs
#
# WARNING: if we force other side to roll to snap.... 
#  we should NOT BE SYNCING ANY more.
# At the moment, it is up to the user to ensure that nothing is going on 
# locally, and future zrep syncs wont just effectively roll forward again
#  on the remote side.
# zrep sync jobs  should probably be halted, until it is decided that
# you want to sync again.
#
# In the future, I should support some kind of "pause" option, for 
#   zrep sync all    to ignore a rolled back filesystem
#
#
synctosnap(){
	typeset srcsnap=$1 destfs=$2 desthost=$3
	typeset newsentlist

	typeset srcfs snapname destsnap
	if [[ "$desthost" == "" ]] ; then
		print ERROR: synctosnap did not receive all required args
		zrep_errquit "args=$@"
	fi
	srcfs=${srcsnap%@*}
	snapname=${srcsnap#*@}
	destsnap=${snapname}

	# Have to enforce OUR syntax. otherwise, any future attempt to
	# continue sync will fail.
	#  ( getlastsnap() wont find it! )
	#
	case $snapname in
		zrep_[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]*)
		: ;;
		*)
			zrep_errquit $srcsnap is not zrep snapshot. Cannot roll with it.
		;;
	esac
		
		

	print Validating remote snap
	zrep_ssh $desthost zfs list -t snapshot $destfs@$destsnap >/dev/null
	if [[ $? -ne 0 ]] ; then
		zrep_errquit  $destfs@$destsnap does not exist. Cannot roll to snap
	fi
	

	print "WARNING: We will be rolling back  $destfs, on $desthost"
	print -n "   to  $snapname, made at: "
	zfs get -H -o value creation $srcsnap
	print ""
	print "All newer snapshots on remote side will be destroyed"
	print "You should have paused ongoing sync jobs for $destfs before continuing"
	print "Continuing in 20 seconds...."
	sleep 10
	print "Continuing in 10 seconds...."
	sleep 10

	zrep_lock_fs $srcfs || zrep_errquit "Cannot lock $srcfs"
	zrep_ssh $desthost zfs rollback -Rr $destfs@$destsnap || zrep_errquit roll failed

	print $desthost:$destfs rolled back successfully to $destsnap
	print Now cleaning up local snapshots

	# need to undo whatever zrep_sync does 
	newsentlist=`getallsnaps $srcfs|sed "1,/@$snapname/d"`
	for snap in $newsentlist ; do
		zfs inherit zrep:sent $snap
	done

	zrep_unlock_fs $srcfs
	
}

# Usage: _snapandsync  fs desthost destfs
# internal routine called by zrep_sync and zrep_failover,
# to do an incremental send.
# You must hold filesystem lock before calling this
# WE DO NOT DO ANY SAFETY OR LOCK CHECKS HERE.
#
# Wil create a new snap on srcfs, and sync it over to given destination
# Sets our 'synced' marker on it as well.
#
_snapandsync(){
	typeset srcfs=$1 desthost=$2 destfs=$3

	typeset sentsnap newsnap snapname



	#srchost=`zfs get -H -o value zrep:src-host $srcfs`
	#if [[ "$srchost" != "$Z_LOCAL_HOST" ]] ; then
	#	zrep_errquit _sync: We are not master for $srcfs
	#fi


	# Find incremental send starting point
	# Do this BEFORE creating new snap, because we should make new snap
	# if we cant do incremental anyway
	sentsnap=`getlastsnapsent $srcfs`
	if [[ "$sentsnap" == "" ]] ; then
		print zrep_sync could not find sent snap for $srcfs. 
		zrep_errquit You must initialize $srcfs for zrep
	fi

	newsnap=`makesnap $srcfs`
	if [[ "$newsnap" == "" ]] ; then
		zrep_errquit zrep_sync could not create new snapshot for $srcfs
	fi

	_sync $srcfs $desthost $destfs $sentsnap $newsnap

}

# called by _snapandsync, and also zrep_synconly
# 	Usage: sourcefs destinationhost destinationfs (lastsent (newsnap))
_sync(){
	typeset srcfs=$1 desthost=$2 destfs=$3
	typeset lastsent=$4 newsnap=$5
	typeset snapname

	if [[ "$lastsent" == "" ]] ; then
		lastsent=`getlastsnapsent $srcfs`
		if [[ "$lastsent" == "" ]] ; then
			print zrep_sync could not find sent snap for $srcfs. 
			zrep_errquit You must initialize $srcfs for zrep
		fi
	fi

	if [[ "$newsnap" == "" ]] ; then
		newsnap=`getlastsnap $srcfs`
		if [[ "$newsnap" == "" ]] ; then
			print zrep_sync could not find sent snap for $srcfs. 
			zrep_errquit You must initialize $srcfs for zrep
		fi
	fi

	if [[ "$newsnap" == "$lastsent" ]] ; then
		print $newsnap already sent
		return 0
	fi

	snapname=${newsnap#*@}

	print sending $newsnap to $desthost:$destfs
	typeset timeinsec=`zrep_gettimeinseconds`
	typeset senttimeprop="zrep:sent=$timeinsec"

	# Note: doing "-o $senttimeprop" sets prop on FILESYSTEM, not snap.
	# So we dont do that usually

	# other than zrep_init, this should be the ONLY place we do a send
	#   Sigh. but now we also do in _refreshpull
	zfs send -I $lastsent $newsnap | 
	   zrep_ssh $desthost zfs recv $destfs
	if [[ $? -ne 0 ]] ; then
		zfs rename ${newsnap} ${newsnap}_unsent
		zrep_errquit Problem doing sync for $newsnap. renamed to ${newsnap}_unsent
	fi

	#Even if we are "old mode", other side may not be.
	# So try newer way first.
	zrep_ssh $desthost zfs set $senttimeprop $destfs@$snapname
	if [[ $? -ne 0 ]] ; then
		print WARNING: setting zrep:sent failed on $desthost:$destfs@$snapname
		print Using fallback. Go patch your system
		zrep_ssh $desthost zfs set zrep:lastsent=${newsnap} $destfs
		zrep_ssh $desthost zfs set zrep:lastsenttime=${timeinsec} $destfs
	fi

	if (( Z_HAS_SNAPPROPS )) ; then
		zfs set $senttimeprop  ${newsnap}
	else
		#note that this is only for old-ZFS compatibility.
		# We dont really want to use this style if possible!
		zfs set zrep:lastsent=${newsnap} $srcfs
		zfs set zrep:lastsenttime=${timeinsec} $srcfs
	fi
}


#User entrypoint, for synconly, which is the pair of snaponly
zrep_synconly(){
#	annoyingly..need to make this almost identical to our current full
#	zrep_sync. but just skipping first steps :(
#	we can skip retries, though.
	typeset srcfs desthost destfs

	# at one point, accept multiple args. But not for now...?
	srcfs=$1

	[[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for synconly

	desthost=`zfs get -H -o value zrep:dest-host $srcfs`
	destfs=`zfs get -H -o value zrep:dest-fs $srcfs`
	if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]];
	then
		zrep_errquit Problem getting zrep properties for fs $srcfs
	fi

	zrep_lock_fs $srcfs
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Failed to acquire zrep lock for $srcfs
	fi

	_sync $srcfs $desthost $destfs || zrep_errquit sync failed for $srcfs

	_expire $srcfs	#dont care so much if this fails

	zrep_unlock_fs $srcfs
}


#zrep_sync
# make a new snapshot and copy it over.
# Usage: zrep_sync [-q quiettime] (all|fs1 .. fsX)
# See workflow.txt 
# Will call synctosnap if a snapshot is given instead of fsname
# Normally, will bail out if another instance of zrep holds lock.
#   -q option says to check last update time of locked filesystems.
#   If sync more recent than given quiettime, then quietly ignore
#   
zrep_sync(){
	typeset srcfs destfs desthost sentsnap newsnap
	typeset quiettime=0

	if [[ "$1" == "-q" ]] ; then
		quiettime="$2"
		shift
		shift
		if (( quiettime < 30 )) ; then
			zrep_errquit "-q must use value greater than 30"
		fi
	fi

	if [[ "$1" == "all" ]] ; then
		set -- `zrep_list_master`
		if [[ "$1" == "" ]] ; then
			# Stay quiet, so we dont spew if in cron
			#print No zrep mastered filesystems found
			exit
		fi
	fi

	if [[ "$1" == "" ]] ; then
		print Error: no filesystems specified for sync >/dev/fd/2
		return 1
	fi

	while [[ "$1" != "" ]] ; do
	srcfs="$1" 

	[[ "$srcfs" == "" ]] && zrep_errquit No fileystem specified for sync

	desthost=`zfs get -H -o value zrep:dest-host $srcfs`
	destfs=`zfs get -H -o value zrep:dest-fs $srcfs`
	if [[ $? -ne 0 ]] || [[ "$desthost" == "-" ]] || [[ "$destfs" == "-" ]];
	then
		zrep_errquit Problem getting zrep properties for fs $srcfs
	fi
	
	case $srcfs in
		*@*)
			synctosnap $srcfs $destfs $desthost
			return
			;;
	esac

	zrep_lock_fs $srcfs

	if [[ $? -ne 0 ]] ; then
		# retry for lock for a while, if (quiettime>0 )

		if ((quiettime==0)); then
			zrep_errquit Cannot lock $srcfs. Cannot continue
		fi
		typeset currtime=`zrep_gettimeinseconds`  snaptime elapsed

		snaptime=`getlastsynctime $srcfs`
		if (( snaptime == 0 )) ; then
			zrep_errquit quiet mode set, but no last snap for $srcfs
		fi

		elapsed=$((currtime - snaptime))
		if ((elapsed > quiettime)) ; then
			print DEBUG: $elapsed seconds have elapsed since last sync of $srcfs
			zrep_errquit quiet time limit of $quiettime seconds exceeded for busy fs $srcfs
		else
			print Quiet mode: skipping busy fs $srcfs at `date`
			return 
		fi
	fi

	_snapandsync $srcfs $desthost $destfs


	# Make this message match what zrep_expire uses..
	print Expiring zrep snaps on $srcfs
	_expire $srcfs

	zrep_unlock_fs $srcfs

	shift

	done

}



# zrep_refresh is a "pull" version of "zrep_sync"
# The concept is a bit of a hack.
# It primarily exists so people can run a secure backup server, that
# has ssh access to all hosts, but not vice versa
#
# Implementation is a bit sketchy. 
# For initial, non-optimal run, perhaps take advantage of
#    ssh host zrep synconly
# to avoid too much duplication of things?
# but will still need to set all the perms n things. Nastyyy..
# The MAIN nastiness, is that all our locks are on the "master" side.
# Which depends on the PID still being there!!
# But if we start now running things on the "slave" side.. 
# There is potential for problems
# Examine critical points and reasons for lock:
#   1. while doing analysis of which snap to send
#   2. to avoid paralel "zfs send"s running.
#   3. for update of timestamp
#
#   We can still wrap #1 and #2 in a single lock call. 
#    (and still on the src side!)
#   The ugly comes when updating zrep:sent. Dont want to update wrong snap!
#   So long as we do some kind of check to see that we're not going
#   backwards when we get lock a second time ... we should be relatively okay.
#   However.. for simplicity... going to just cross fingers and wrap
#   all three in single remote lock call, through _refreshpull
#
zrep_refresh(){
	typeset srcfs destfs desthost newsnap newseq master

	# for now, just handle ONE arg, not multiple fs list

	destfs="$1"
	if [[ "$1" == "" ]] ; then
		print Error: no filesystems specified for refresh >/dev/fd/2
		return 1
	fi

	master=`zfs get -H -o value -s local zrep:master $destfs`
	if [[ "$master" == "yes" ]] ; then
		zrep_errquit Sorry, you cant run refresh on a master mode fs $destfs
	fi

	srchost=`zfs get -H -o value zrep:src-host $destfs`
	srcfs=`zfs get -H -o value zrep:src-fs $destfs`

	zrep_lock_fs $destfs
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Cannot lock $destfs. Cannot continue
	fi

	print DEBUG: refresh step 1: Going to $srchost to snapshot $destfs
	newsnap=`zrep_ssh $srchost $ZREP_PATH snaponly $srcfs`
	if [[ $? -ne 0 ]] ; then
		zrep_errquit snap of src $srcfs on $srchost failed
	fi
	# yes, MORE paranoia..
	case $newsnap in
		*@zrep_*)
			newseq=${newsnap#*@}
			;;
		*)
			zrep_errquit Unrecognized output from src snap. Cannot continue
			;;					
	esac
	
	
	typeset timeinsec=`zrep_gettimeinseconds`
	typeset senttimeprop="zrep:sent=$timeinsec"

	print DEBUG: refresh step 2: Pulling $newsnap

	zrep_ssh $srchost $ZREP_PATH _refreshpull $newsnap |
	  zfs recv $destfs
	if [[ $? -ne 0 ]] ; then
		zrep_errquit Unforseen error pulling snapshot $newsnap from $srchost
	fi

	zfs set $senttimeprop $destfs@$newseq
	if [[ $? -ne 0 ]] ; then
		print WARNING: expected local copy $destfs@newseq does not exist >/dev/fd/2
	fi

	zrep_unlock_fs $destfs

}

# Hidden command-line option for "zrep refresh"
# This is the "remote call" to support zrep refresh
#    ( aka  zrep_refresh  )
# In principle, its kinda like "zrep expire" being callable by
# both the user, and the program itself. 
# However, this routine is definitely not supposed to be user visible
#   .. eh... maybe someday. but initial design is "private"
_refreshpull(){
	typeset fs snapname lastsent latest timeinsec senttimeprop

	snapname="$1"
	fs=${snapname%@*}

	print DEBUG: _refreshpull: snapname=$snapname, fs=$fs >/dev/fd/2

	zrep_lock_fs $fs
	if [[ $? -ne 0  ]] ; then
		zrep_errquit Could not lock $fs
	fi

	#We should now;
	# 1. compare to latest snap. quit if not latest
	# 2. get timestamp
	# 3. trigger a zfs send
	# 4.  set timestamp if no errors.
	#    I think it is reasonable to presume that if the receive failed,
	#    we will see an error by the pipe blowing up.
	#

	lastsent=`getlastsnapsent $fs`
	if [[ "$lastsent" == "" ]] ; then
		zrep_errquit Canthappen: _refreshpull cant findlastsent snap
	fi
	latest=`getlastsnap $fs`

	if [[ "$latest" != "$snapname" ]] ; then
		zrep_errquit Sync error: $snapname is not latest snap for $fs
	fi

	timeinsec=`zrep_gettimeinseconds`
	senttimeprop="zrep:sent=$timeinsec"

	if (( Z_HAS_SNAPPROPS ==0)) ; then
		zrep_errquit Error: we currently only support modern ZFS that allows setting props on snaps
	fi

	zfs send -I $lastsent $latest

	if [[ $? -ne 0 ]] ; then
		zrep_errquit Some kind of error during sending. Bailing out of _refreshpull
	fi


	zfs set $senttimeprop $latest
	zrep_unlock_fs $fs
}

# _expire: 
#   get rid of "old" snapshots for a specifically named filesystem
#
# Note0: you must hold local(master) fs lock first
#
# Note1: expire BOTH SIDES, if we are master
# Keep in mind that sometimes master and dest are on same system
# 
# Note2: Be sure to NEVER delete most recent sent snapshot!!

# INTERNAL routine. For external-facing routine, see zrep_expire
_expire(){
	typeset savecount currcount lastsent remotehost remotefs sanity
	typeset tmpfile=/var/run/zrep_expire.$$
	typeset local=0 master

	if [[ "$1" == "-L" ]] ; then
		local=1;
		shift
	fi

	master=`zfs get -H -o value -s local zrep:master $1`

	zrep_has_fs_lock $1 || zrep_errquit zrep_expire Internal Err caller did not hold fs lock

	savecount=`zfs get -H -o value zrep:savecount $1`
	# do not use (()) in case value unset
	if [[ $savecount < 1 ]] ; then
		zrep_errquit zrep:savecount on $1 set to improper value $savecount
	fi

	if [[ "$master" == "yes" ]] ; then
		lastsent=`getlastsnapsent $1`
		if [[ "$lastsent" == "" ]] ; then
			zrep_errquit corrupted zrep data: no last sent detected. Stopping expire
		fi

		getallsnaps $1 |egrep -v $lastsent >$tmpfile
		savecount=$((savecount-1))
	else
		getallsnaps $1 >$tmpfile
	fi
	currcount=`wc -l < $tmpfile`

	if ((currcount > savecount )) ; then
		currcount=$((currcount - savecount))

		head -$currcount $tmpfile >$tmpfile.2
		mv $tmpfile.2 $tmpfile
		for snap in `cat $tmpfile` ; do
			print DEBUG: expiring $snap
			zfs destroy $snap
		done
	fi
	rm $tmpfile
	

	if [[ "$master" != "yes" ]] || ((local ==1)) ; then
		#This fs is dest fs. We are done.
		return
		#otherwise, go expire on remote side as well
	fi

	remotehost=`zfs get -H -o value zrep:dest-host $1`
	remotefs=`zfs get -H -o value zrep:dest-fs $1`
	print Also running expire on $remotehost:$remotefs now...
	sanity=`zrep_ssh $remotehost zfs get -H -o value -s local zrep:master $remotefs`

	# Normally, dont quit on error. But this is super-bad.
	if [[ "$sanity" == "yes" ]] ; then
		zrep_errquit "Remote side also marked as master ** $remotehost:$remotefs"
	fi

	zrep_ssh $remotehost $ZREP_PATH expire $remotefs ||print REMOTE expire failed
}


# top-level user-facing routine.
# expire old snaps for some or all zrep filesystems.
# Different ways of calling:
#   zrep expire all	Run expire on all zrep fs
#   zrep expire		Run expire on zrep fs we are master for, plus remote
#   zrep expire	-L	Run expire on zrep fs we are master for. SKIP remote
#   zrep expire	 fs ..	Run expire only on fs, plus remote if it is a master
#   zrep expire	-L fs 	Run expire only on fs. Skip remote
#
# If no arg given, expire only filesystems we are master for
# If "all" given, expire literally all.
#
zrep_expire()
{
	typeset local
	if [[ "$1" == "-L" ]] ; then
		local="-L"
		shift
	fi

	if [[ "$1" == "all" ]] ; then
		set -- `zrep_list`
	elif [[ "$1" == "" ]] ; then
		set -- `zrep_list_master`
	fi

	# Note: we should continue if we hit problems with an individual
	# filesystem. Otherwise we risk letting server selfdestruct fill
	# over one troublesome filesystem
	#
	while [[ "$1" != "" ]] ; do
		zrep_lock_fs $1
		print Expiring zrep snaps on $1
		 _expire $local $1 || print WARNING: expire failed for $1
		zrep_unlock_fs $1
		shift
	done
	
}




# run this on 'master' side, to make other side master
zrep_failover(){
	typeset local=0 fs snap="" remotehost remotefs check

	if [[ "$1" == "-L" ]] ; then
		local=1
		shift
	fi

	if [[ "$1" == "" ]] ; then
		usage
		exit 1
	fi

	zfs list $1 >/dev/null || zrep_errquit invalid filesystem $1
	check=`zfs get -H -o value -s local zrep:master $1`
	if [[ "$check" != "yes" ]] ; then
		zrep_errquit $1 not master. Cannot fail over
	fi

	fs="$1"
	
	case $fs in
		*@*)
			snap=$fs
			fs=${srcsnap%@*}
		;;
	esac

	zrep_lock_fs $fs ||zrep_errquit could not lock $fs

	remotehost=`zfs get -H -o value zrep:dest-host $fs`
	remotefs=`zfs get -H -o value zrep:dest-fs $fs`

	print setting readonly on local $fs, then syncing
	zfs set readonly=on $fs

	if ((local ==1)) ; then
		print failover for $1 in LOCAL mode
		if [[ "$snap" == "" ]] ; then
			snap=`getlastsnapsent $1`
			zfs list $1 >/dev/null ||
				zrep_errquit No last synced snap found for $1. Cannot fail over
			print Rolling back to last sync $snap
		else
			print Rolling back to specified snap $snap
		fi
		zfs rollback -Rr $snap ||zrep_errquit Rollback to $snap failed

	else

		## Need to sync both sides before mode switch!
		## If named snap, roll back.
		## otherwise, "roll forward" by doing one last sync

		if [[ "$snap" != "" ]] ; then
			typeset snapname
			snapname=${snap#*@}

			print Rolling back to local $snap
			zfs rollback -Rr $snap || zrep_errquit Rollback to $snap failed
			print Rolling back $remotehost to $remotefs@$snapname
			zrep_ssh $remotehost zfs rollback $remotefs@$snapname ||
				zrep_errquit remote rollback failed

		else
			# makes new snapshot, and syncs 
			_snapandsync $fs $remotehost $remotefs || zrep_errquit final sync failed. failover failed.
		fi


	fi

	print Reversing master properties for $Z_LOCAL_HOST:$fs

	zfs set zrep:dest-fs=$fs $fs	
	zfs set zrep:dest-host=$Z_LOCAL_HOST $fs
	zfs set zrep:src-fs=$remotefs $fs	
	zfs set zrep:src-host=$remotehost $fs

	zfs inherit zrep:master $fs
	
	zrep_unlock_fs $fs

	if (( local ==0)) ;then
		print Setting master on $remotehost:$remotefs
		zrep_ssh $remotehost $ZREP_PATH takeover -L $remotefs
	fi
	
}

# run this on 'dest' side, to promote it to master
zrep_takeover(){
	typeset fs snap remotehost remotefs check local=0

	if [[ "$1" == "-L" ]] ; then
		local=1
		shift
	fi
	
	if [[ "$1" == "" ]] ; then
		usage
		exit 1
	fi

	fs="$1"

	zfs list $fs >/dev/null || zrep_errquit invalid filesystem $fs

	check=`zfs get -H -o value -s local zrep:master $fs`
	if [[ "$check" = "yes" ]] ; then
		zrep_errquit $fs is already master. Cannot takeover
	fi


	remotehost=`zfs get -H -o value zrep:src-host $fs`
	remotefs=`zfs get -H -o value zrep:src-fs $fs`


	if (( local == 0 )) ; then

		print starting failover from remote side $remotehost
		zrep_ssh $remotehost $ZREP_PATH failover $remotefs
		exit $?
	fi

	# If here, we must be in local mode.
	# So... just set properties!
	# (and roll back, if desired)

	case $fs in
		*@*)
			snap=$fs
			fs=${srcsnap%@*}
		;;
	esac



	zrep_lock_fs $fs
	zfs inherit readonly $fs

	if [[ "$snap" != "" ]] ; then
		print "WARNING: Before takeover, we will be rolling $fs"
		print -n "   to  $snapname, made at: "
		zfs get -H -o value creation $snap
		print ""
		print "All newer snapshots will be destroyed"
		print Continuing in 10 seconds...
		sleep 10

		zfs rollback -Rr $snap || zrep_errquit Rollback to $snap failed
	fi

	print Setting master properties for $Z_LOCAL_HOST:$fs

	zfs set zrep:src-fs=$fs $fs	
	zfs set zrep:src-host=$Z_LOCAL_HOST $fs
	zfs set zrep:dest-fs=$remotefs $fs	
	zfs set zrep:dest-host=$remotehost $fs

	zfs set zrep:master=yes $fs
	
	zrep_unlock_fs $fs
	

}

######## zrep_top continues here

usage(){
	print zrep v${ZREP_VERSION}: a program to replicate a zfs filesystem to another
	print in an ongoing basis.
	print More documentation at a later date.
	print "  Philip Brown, 2012"
	print ""
	print Simple usage summary:
	print 'zrep (init|-i) ZFS/fs remotehost remoteZFSpool/fs'
	print 'zrep (sync|-S) [-q seconds] ZFS/fs'
	print 'zrep (sync|-S) [-q seconds] all'
	print 'zrep (sync|-S) ZFS/fs@snapshot    -- temporary retroactive sync'
	print 'zrep (status|-s) [-v] [(-a|ZFS/fs)]'
	print 'zrep refresh ZFS/fs               -- pull version of sync'
	print 'zrep (list|-l) [-v]'
	print 'zrep (expire|-e) [-L] (ZFS/fs ...)|(all)|()'
	print 'zrep (changeconfig|-C) ZFS/fs remotehost remoteZFSpool/fs'
	print 'zrep failover [-L] ZFS/fs'
	print 'zrep takeover [-L] ZFS/fs'
	print 'zrep clear ZFS/fs  -- REMOVE ZREP CONFIG AND SNAPS FROM FILESYSTEM'
	print
	print '   -q option says to Quietly ignore locked filesystems that have synced'
	print '      more recently than the given amount of seconds'
	print
	print 'Paired commands for high-transaction systems:'
	print '   zrep snaponly ZFS/fs'
	print '   zrep synconly ZFS/fs'
	print 'The above two commands split the simple sync subcommand, into two'
	print 'separate steps, so that a database, etc. may resume while the sync'
	print 'completes in the background'
	print ''
	print ' More detailed examples can be found at:'
	print http://www.bolthole.com/solaris/zrep/zrep.documentation.html
}

case "$1" in
	"")
		usage
		;;
	clear)
		shift
		zrep_clear $1
		;;
	expire|-e)
		shift
		zrep_expire "$@"
		;;
	init|-i)
		shift
		zrep_init "$@"
		;;
	changeconfig|-C)
		shift
		zrep_changeconfig "$@"
		;;
	sync|-S)
		shift
		zrep_sync "$@"
		;;
	snaponly)
		shift
		zrep_snaponly "$@"
		;;
	synconly)
		shift
		zrep_synconly "$@"
		;;
	refresh)	# yes keep this in this order
		shift
		zrep_refresh "$@"
		;;
	status|-s)
		shift
		zrep_status "$@"
		;;
	list|-l)
		shift
		zrep_list "$@"
		;;
	failover)
		shift
		zrep_failover "$@"
		;;
	takeover)
		shift
		zrep_takeover "$@"
		;;
		
	_refreshpull)  # Secret option DO NOT PUT IN USAGE!!
		shift
		_refreshpull $1
		;;
	*)
		usage
		;;
esac


