#! /usr/bin/env bash
#
#   Copyright (c) 2015 Nat! - Mulle kybernetiK
#   All rights reserved.
#
#   Redistribution and use in source and binary forms, with or without
#   modification, are permitted provided that the following conditions are met:
#
#   Redistributions of source code must retain the above copyright notice, this
#   list of conditions and the following disclaimer.
#
#   Redistributions in binary form must reproduce the above copyright notice,
#   this list of conditions and the following disclaimer in the documentation
#   and/or other materials provided with the distribution.
#
#   Neither the name of Mulle kybernetiK nor the names of its contributors
#   may be used to endorse or promote products derived from this software
#   without specific prior written permission.
#
#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
#   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
#   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
#   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
#   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
#   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
#   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
#   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
#   POSSIBILITY OF SUCH DAMAGE.
#

MULLE_BOOTSTRAP_FETCH_SH="included"

#
# this script installs the proper git clones into "clones"
# it does not to git subprojects.
# You can also specify a list of "brew" dependencies. That
# will be third party libraries, you don't tag or debug
#
#
# ## NOTE ##
#
# There is a canonical argument passing scheme, which gets passed to and
# forwarded by most function
#
# reposdir="$1"   # ususally .bootstrap.repos
# name="$2"       # name of the clone
# url="$3"        # URL of the clone
# branch="$4"     # branch of the clone
# scm="$5"        # scm to use for this clone
# tag="$6"        # tag to checkout of the clone
# stashdir="$7"     # stashdir of this clone (absolute or relative to $PWD)
#

fetch_usage()
{
   cat <<EOF >&2
Usage:
   ${MULLE_EXECUTABLE} ${COMMAND} [options] [repositories]

   You can specify the names of the repositories to update.

Commands:
   install  :  clone or symlink non-exisiting repositories and other resources
   update   :  execute a "fetch" in already fetched repositories
   upgrade  :  execute a "pull" in fetched repositories

Options:
   --caches            :  use CACHES_PATH to locate local repositories
   --check-usr-local   :  check /usr/local for duplicates
   --embedded-only     :  fetch embedded repositories only

   --symlinks          :  allow symlinking instead of cloning
   --embedded-symlinks :  allow embedded symlinks (very experimental)
   --follow-symlinks   :  follow symlinks when updating (not recommended)
   --no-caches         :  don't use caches. Useful to counter flag -y
   --no-symlinks       :  don't create symlinks. Useful to counter flag -y

EOF

   local  repositories

   repositories="`all_repository_names`"
   if [ -z "${repositories}" ]
   then
      echo "Currently available repositories are:"
      echo "${repositories}" | sed 's/^/   /'
   fi
   exit 1
}


assert_sane_parameters()
{
   local  empty_reposdir_is_ok="$1"

   [ ! -z "${empty_reposdir_is_ok}" -a -z "${reposdir}" ] && internal_fail "parameter: reposdir is empty"
   [ -z "${empty_reposdir_is_ok}" -a ! -d "${reposdir}" ] && internal_fail "parameter: reposdir does not exist ($reposdir)"

   [ -z "${url}" ]      && internal_fail "parameter: url is empty"
   [ -z "${name}" ]     && internal_fail "parameter: name is empty"
   [ -z "${stashdir}" ] && internal_fail "parameter: stashdir is empty"

   :
}

#
# future, download tarballs...
# we check for existance during fetch, but install during build
#
check_tars()
{
   local tarballs
   local tar

   log_fluff "Looking for tarballs"

   tarballs="`read_root_setting "tarballs" | sort | sort -u`"
   if [ ! -z "${tarballs}" ]
   then
      [ -z "${DEFAULT_IFS}" ] && internal_fail "IFS fail"
      IFS="
"
      for tar in ${tarballs}
      do
         IFS="${DEFAULT_IFS}"

         if [ ! -f "$tar" ]
         then
            fail "tarball \"$tar\" not found"
         fi
         log_fluff "tarball \"$tar\" found"
      done
      IFS="${DEFAULT_IFS}"

   else
      log_fluff "No tarballs found"
   fi
}


log_action()
{
   local action="$1" ; shift

   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   assert_sane_parameters "empty reposdir is ok"

   local info

   if [ -L "${url}" ]
   then
      info=" symlinked "
   else
      info=" "
   fi

   log_fluff "Perform ${action}${info}${url} into ${stashdir} ..."
}


#
###
#
link_command()
{
#   local reposdir="$1"  # ususally .bootstrap.repos
#   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
#   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   assert_sane_parameters "empty reposdir is ok"

   local absolute

   absolute="`read_config_setting "absolute_symlinks" "NO"`"
   if ! exekutor create_symlink "${url}" "${stashdir}" "${absolute}"
   then
      return 1
   fi

   local branchlabel

   branchlabel="branch"
   if [ -z "${branch}" -a ! -z "${tag}" ]
   then
      branchlabel="tag"
      branch="${tag}"
   fi

   if [ "${branch}" != "master" ]
   then
      log_warning "The intended ${branchlabel} ${C_RESET_BOLD}${branch}${C_WARNING} \
will be ignored, because the repository is symlinked.
If you want to checkout this ${branchlabel} do:
   ${C_RESET_BOLD}(cd ${stashdir}; git checkout ${GITOPTIONS} \"${branch}\" )${C_WARNING}"
   fi
}


can_symlink_it()
{
   local  directory="$1"

   if [ "${OPTION_ALLOW_CREATING_SYMLINKS}" != "YES" ]
   then
      log_fluff "Can't symlink it, because forbidden"
      return 1
   fi

   case "${UNAME}" in
      minwgw)
         log_fluff "Can't symlink it, because symlinking is unavailable on this platform"
         return 1
      ;;
   esac

   if git_is_repository "${directory}"
   then
       # if bare repo, we can only clone anyway
      if git_is_bare_repository "${directory}"
      then
         log_info "${directory} is a bare git repository. So cloning"
         log_info "is the only way to go."
         return 1
      fi
   else
      log_info "${directory} is not a git repository (yet ?)"
      log_info "So symlinking is the only way to go."
   fi

  return 0
}


_search_for_git_repository_in_cache()
{
   local directory
   local name
   local branch

   [ $# -ne 3 ] && internal_fail "fail"

   directory="$1"
   name="$2"
   branch="$3"

   local found

   if [ "${MULLE_FLAG_LOG_CACHE}" = "YES" ]
   then
      log_trace "Checking cache path \"${directory}\""
   fi

   if [ ! -z "${branch}" ]
   then
      found="${directory}/${name}.${branch}"
      log_fluff "Looking for \"${found}\""

      if [ -d "${found}" ]
      then
         log_fluff "Found \"${name}.${branch}\" in \"${directory}\""

         echo "${found}"
         return
      fi
   fi

   found="${directory}/${name}.git"
   log_fluff "Looking for \"${found}\""
   if [ -d "${found}" ]
   then
      log_fluff "Found \"${name}.git\" in \"${directory}\""

      echo "${found}"
      return
   fi

   found="${directory}/${name}"
   log_fluff "Looking for \"${found}\""
   if [ -d "${found}" ]
   then
      log_fluff "Found \"${name}\" in \"${directory}\""

      echo "${found}"
      return
   fi
}


search_for_git_repository_in_caches()
{
   local found
   local directory
   local realdir
   local curdir

   if [ "${MULLE_FLAG_LOG_CACHE}" = "YES" -a -z "${CACHES_PATH}" ]
   then
      log_trace "CACHES_PATH is empty"
   fi

   curdir="`pwd -P`"
   IFS=":"
   for directory in ${CACHES_PATH}
   do
      IFS="${DEFAULT_IFS}"

      if [ ! -d "${directory}" ]
      then
         if [ "${MULLE_FLAG_LOG_CACHE}" = "YES" ]
         then
            log_trace2 "Cache path \"${realdir}\" does not exist"
         fi
         continue
      fi

      realdir="`realpath "${directory}"`"
      if [ "${realdir}" = "${curdir}" ]
      then
         fail "config setting \"search_path\" mistakenly contains \"${directory}\", which is the current directory"
      fi

      found="`_search_for_git_repository_in_cache "${realdir}" "$@"`" || exit 1
      if [ ! -z "${found}" ]
      then
         symlink_relpath "${found}" "${ROOT_DIR}"
         return
      fi
   done

   IFS="${DEFAULT_IFS}"
   return 1
}


_search_for_archive_in_caches()
{
   local directory="$1"
   local name="$2"
   local filename="$3"

   [ $# -ne 3 ] && internal_fail "fail"

   local found

   found="${directory}/${name}-${filename}"
   log_fluff "Looking for \"${found}\""
   if [ -f "${found}" ]
   then
      log_fluff "Found \"${name}\" in \"${directory}\" as \"${found}\""

      echo "${found}"
      return
   fi

   found="${directory}/${filename}"
   log_fluff "Looking for \"${found}\""
   if [ -f "${found}" ]
   then
      log_fluff "Found \"${name}\" in \"${directory}\" as \"${found}\""

      echo "${found}"
      return
   fi
}


search_for_archive_in_caches()
{
   local name="$1"
   local filename="$2"

   local found
   local directory

   IFS=":"
   for directory in ${CACHES_PATH}
   do
      IFS="${DEFAULT_IFS}"

      found="`_search_for_archive_in_caches "${directory}" "${name}" "${filename}"`" || exit 1
      if [ ! -z "${found}" ]
      then
         found="`absolutepath "${found}"`"
         echo "file:///${found}"
         return
      fi
   done

   IFS="${DEFAULT_IFS}"
   return 1
}


mkdir_stashparent_if_missing()
{
   local stashdir="$1"

   local stashparent

   stashparent="`dirname -- "${stashdir}"`"
   case "${stashparent}" in
      ""|"\.")
      ;;

      *)
         mkdir_if_missing "${stashparent}"
         echo "${stashparent}"
      ;;
   esac
}


get_cache_item()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone, extensionless
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   if [ "${OPTION_ALLOW_SEARCH_CACHES}" = "NO" ]
   then
      log_fluff "Not searching caches because --no-caches"
      return
   fi

   case "${scm}" in
      git*)
         search_for_git_repository_in_caches "${name}" "${branch}"
      ;;

      tar*|zip*)
         search_for_archive_in_caches "${name}" "`basename -- "${url}"`"
      ;;

      *)
         log_fluff "Not searching caches because scm is \"${scm}\""
      ;;
   esac
}


clone_or_symlink()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone, extensionless
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)


   assert_sane_parameters "empty reposdir is ok"

   [ $# -le 7 ] || internal_fail "too many parameters"

   local operation
   local scmflagsdefault

   case "${scm}" in
      git*)
         operation="git_clone"
      ;;

      zip*)
         operation="zip_unpack"
      ;;

      tar*)
         operation="tar_unpack"
      ;;

      svn*)
         operation="svn_checkout"
      ;;

      *)
         fail "Unknown scm system \"${scm}\""
      ;;
   esac

   local SCM_OPTIONS

   #
   # later: make it so tar?shasum256=djhdjhdfdh
   # and check that the archive is correct
   #
   extra="`echo "${scm}" | sed -n 's/^[^?]*\?\(.*\)/\1/p'`"
   if [ ! -z "${extra}" ]
   then
      log_fluff "Parsed SCM_OPTIONS as \"${extra}\""
      SCM_OPTIONS="${extra}"
   fi

   local stashparent

   stashparent="`mkdir_stashparent_if_missing "${stashdir}"`"

   local found
   local script

   script="`find_root_setting_file "bin/clone.sh"`"

   if [ ! -z "${script}" ]
   then
      run_script "${script}" "$@"
      return $?
   fi

   case "${url}" in
      /*)
         if can_symlink_it "${directory}"
         then
            operation=link_command
         fi
      ;;

      #
      # don't move up using url
      #
      */\.\./*|\.\./*|*/\.\.|\.\.)
         internal_fail "Faulty url \"${url}\" should have been caught before"
      ;;

      *)
         found="`get_cache_item "${reposdir}" \
                                "${name}" \
                                "${url}" \
                                "${branch}" \
                                "${scm}" \
                                "${tag}" \
                                "${stashdir}"`"

         if [ ! -z "${found}" ]
         then
            url="${found}"

            case "${scm}" in
               git)
                  if can_symlink_it "${url}"
                  then
                     operation=link_command
                  fi
               ;;
            esac

            case "${operation}" in
               link_command)
                 log_info "Using symlink to cached item \"${found}\""
               ;;

               *)
                 log_info "Using cached item \"${found}\""
               ;;
            esac
         fi
      ;;
   esac

   local rval

   if ! "${operation}" "${reposdir}" \
                       "${name}" \
                       "${url}" \
                       "${branch}" \
                       "${scm}" \
                       "${tag}" \
                       "${stashdir}"
   then
      log_debug "Clone \"${operation}\" failed"
      return 1
   fi

   if [ "${DONT_WARN_SCRIPTS}" != "YES" ]
   then
      [ -z "${MULLE_BOOTSTRAP_WARN_SCRIPTS_SH}" ] && . mulle-bootstrap-warn-scripts.sh

      warn_scripts_main "${stashdir}/.bootstrap" "${stashdir}" || fail "Ok, aborted"  #sic
   fi

   if [ "${operation}" = "link_command" ]
   then
      log_debug "Symlink \"${operation}\" successful"
      return 2
   fi

   log_debug "Clone \"${operation}\" successful"
   return 0
}

##
## CLONE
##
clone_repository()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   [ $# -eq 7 ] || internal_fail "fail"

   log_action "clone" "$@"

   assert_sane_parameters "empty is ok"

   if [ "${OPTION_CHECK_USR_LOCAL_INCLUDE}" = "YES" ] && has_usr_local_include "${name}"
   then
      log_info "${C_MAGENTA}${C_BOLD}${name}${C_INFO} is a system library, so not fetching it"
      return 1
   fi

   if [ -e "${stashdir}" ]
   then
      if [ "${url}" = "${stashdir}" ]
      then
         if is_master_bootstrap_project
         then
            is_minion_bootstrap_project "${stashdir}" || fail "\"${stashdir}\" \
should be a minion but it isn't.
Suggested fix:
   ${C_RESET}${C_BOLD}cd \"${stashdir}\" ; ${MULLE_EXECUTABLE} defer \"\
`symlink_relpath "${PWD}" "${stashdir}"`\
\""
            log_info "${C_MAGENTA}${C_BOLD}${name}${C_INFO} is a minion, so cloning is skipped"
            return 1
         fi
      fi
      bury_stash "${reposdir}" "${name}" "${stashdir}"
   fi

   clone_or_symlink "$@"   # pass thru rval
}


##
## CHECKOUT
##
checkout_repository()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   log_action "checkout" "$@"

   local operation

   case "${scm}" in
      git)
         operation="git_checkout"
      ;;
      svn)
         operation="svn_checkout"
      ;;
      *)
         fail "Unknown scm system \"${scm}\""
      ;;
   esac

   script="`find_build_setting_file "${name}" "bin/checkout.sh"`"
   if [ ! -z "${script}" ]
   then
      run_script "${script}" "$@"
   else
      "${operation}" "$@"
   fi
}


##
## UPDATE
## this like git fetch, does not update repository
##
update_repository()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   log_action "update" "$@"

   local operation

   case "${scm}" in
      git)
         operation="git_fetch"
      ;;
      svn)
         return
      ;;
      *)
         fail "Unknown scm system \"${scm}\""
      ;;
   esac

   script="`find_build_setting_file "${name}" "bin/update.sh"`"
   if [ ! -z "${script}" ]
   then
      run_script "${script}" "$@"
   else
      "${operation}" "$@"
   fi
}


##
## UPGRADE
## This is a pull

upgrade_repository()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   log_action "upgrade" "$@"

   local operation

   case "${scm}" in
      git)
         operation="git_pull"
      ;;
      svn)
         return
      ;;
      *)
         fail "Unknown scm system \"${scm}\""
      ;;
   esac

   script="`find_build_setting_file "${name}" "bin/upgrade.sh"`"
   if [ ! -z "${script}" ]
   then
      run_script "${script}" "$@"
   else
      "${operation}" "$@"
   fi
}


#
# Walk repositories with a callback function
#
_update_operation_walk_repositories()
{
   local operation="$1"

   local permissions

   permissions="minion"
   walk_auto_repositories "repositories"  \
                          "${operation}" \
                          "${permissions}" \
                          "${REPOS_DIR}"

   permissions="minion"
   walk_auto_repositories "additional_repositories"  \
                          "${operation}" \
                          "${permissions}" \
                          "${REPOS_DIR}"
}


_update_operation_walk_embedded_repositories()
{
   local operation="$1"

   local permissions

   permissions="minion"
   #
   # embedded repositories can't be symlinked by default
   # embedded repositories are by default not put into
   # stashes (for backwards compatibility)
   #
   (
      STASHES_ROOT_DIR=""
      STASHES_DEFAULT_DIR=""
      OPTION_ALLOW_CREATING_SYMLINKS="${OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS}" ;

      walk_auto_repositories "embedded_repositories"  \
                             "${operation}" \
                             "${permissions}" \
                             "${EMBEDDED_REPOS_DIR}"
   ) || exit 1
}


_update_operation_walk_deep_embedded_auto_repositories()
{
   local operation="$1"

   local permissions

   permissions="minion"

   (
      OPTION_ALLOW_CREATING_SYMLINKS="${OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS}" ;

      walk_deep_embedded_auto_repositories "${operation}" \
                                           "${permissions}"
   ) || exit 1
}


##
## UPDATE
##
update_repositories()
{
   _update_operation_walk_repositories "update_repository"
}


update_embedded_repositories()
{
   _update_operation_walk_embedded_repositories "update_repository"
}


update_deep_embedded_repositories()
{
   _update_operation_walk_deep_embedded_auto_repositories "update_repository"
}


##
## UPGRADE
##
upgrade_repositories()
{
   _update_operation_walk_repositories "upgrade_repository"
}


upgrade_embedded_repositories()
{
   _update_operation_walk_embedded_repositories "upgrade_repository"
}


upgrade_deep_embedded_repositories()
{
   _update_operation_walk_deep_embedded_auto_repositories "upgrade_repository"
}



##
##
##
required_action_for_clone()
{
   local newclone="$1" ; shift

   local newreposdir="$1"  # ususally .bootstrap.repos
   local newname="$2"      # name of the clone
   local newurl="$3"       # URL of the clone
   local newbranch="$4"    # branch of the clone
   local newscm="$5"       # scm to use for this clone
   local newtag="$6"       # tag to checkout of the clone
   local newstashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   local clone

   clone="`clone_of_repository "${reposdir}" "${name}"`"
   if [ -z "${clone}" ]
   then
      log_fluff "${url} is new"
      echo "clone"
      return
   fi

   if [ "${clone}" = "${newclone}" ]
   then
      if [ -e "${newstashdir}" ]
      then
         log_fluff "URL ${url} repository line is unchanged"
         return
      fi

      log_fluff "\"${newstashdir}\" is missing, reget."
      echo "clone"
      return
   fi

   local reposdir

   reposdir="${newreposdir}"

   local name
   local url
   local branch
   local scm
   local tag
   local stashdir

   parse_clone "${clone}"

   log_debug "Change: \"${clone}\" -> \"${newclone}\""

   if [ "${scm}" != "${newscm}" ]
   then
      if ! [ "${scm}" = "symlink" -a "${newscm}" = "git" ]
      then
         log_fluff "SCM has changed from \"${scm}\" to \"${newscm}\", need to refetch"

         if [ "${scm}" != "minion" ]
         then
            log_fluff "Not removing \"${stashdir}\" because it's a minion"
            echo "remove"
         fi

         echo "clone"
         return
      fi
   fi

   if [ "${stashdir}" != "${newstashdir}" ]
   then
      if [ -e "${newstashdir}" ]
      then
         log_fluff "Destination \"${newstashdir}\" already exists. Remove old."
         echo "remove"

         # detect unlucky hacque
         if is_minion_bootstrap_project "${newstashdir}"
         then
            fail "\"${stashdir}\" was renamed to minion \"${newstashdir}\""
         fi
      else
         log_fluff "Destination has changed from \"${stashdir}\" to \"${newstashdir}\", need to move"

         local oldstashdir

         oldstashdir="`get_old_stashdir "${reposdir}" "${name}"`"
         if [ "${oldstashdir}" != "${newstashdir}" ]
         then
            if [ -d "${oldstashdir}" ]
            then
               echo "move ${oldstashdir}"
            else
               log_warning "Can't find ${name} in ${oldstashdir}. Will clone again"
               echo "clone"
            fi
         fi
      fi
   fi

   #
   # if scm is not git, don't try to be clever
   #
   if [ "${scm}" = "symlink" ]
   then
      if [ -e "${newstashdir}" ]
      then
         log_fluff "\"${stashdir}\" is symlink. Ignoring possible differences."
         return
      fi

      log_fluff "\"${newstashdir}\" is missing, reget."
      echo "clone"
      return
   fi

   if [ "${branch}" != "${newbranch}" ]
   then
      log_fluff "Branch has changed from \"${branch}\" to \"${newbranch}\", need to fetch"
      echo "upgrade"
   fi

   if [ "${tag}" != "${newtag}" ]
   then
      log_fluff "Tag has changed from \"${tag}\" to \"${newtag}\", need to check-out"
      echo "checkout"
   fi

   if [ "${url}" != "${newurl}" ]
   then
      log_fluff "URL has changed from \"${url}\" to \"${newurl}\", need to set remote url and fetch"
      echo "set-remote"
      echo "upgrade"
   fi
}


get_old_stashdir()
{
   local reposdir="$1"
   local name="$2"

   [ -z "${reposdir}" ] && internal_fail "reposdir empty"
   [ -z "${name}" ]     && internal_fail "reposdir empty"

   local oldstashdir
   local oldparent

   oldclone="`clone_of_repository "${reposdir}" "${name}"`"

   [ -z "${oldclone}" ] && fail "Old clone information for \"${name}\" in \"${reposdir}\" is missing"

   url="`_url_part_from_clone "${oldclone}"`"
   name="`_canonical_clone_name "${url}"`"
   dstdir="`_dstdir_part_from_clone "${oldclone}"`"

   oldparent="`parentclone_of_repository "${reposdir}" "${name}"`"
   if [ ! -z "${oldparent}" ]
   then
      # figure out what the old stashdir prefix was and remove it
      oldprefix="`_dstdir_part_from_clone "${oldparent}"`"
      dstdir="${dstdir#$oldprefix/}"
   fi

   oldstashdir="`computed_stashdir "${url}" "${name}" "${dstdir}"`"

   echo "${oldstashdir}"
}


auto_update_minions()
{
   local minions="$1"

   local minion

   log_debug ":auto_update_minions: (${minions})"

   IFS="
"
   for minion in ${minions}
   do
      IFS="${DEFAULT_IFS}"

      if [ -z "${minion}" ]
      then
         continue
      fi

      log_debug "${C_INFO}Updating minion \"${minion}\" ..."

      [ ! -d "${minion}" ] && fail "Minion \"${minion}\" is gone"

      bootstrap_auto_update "${minion}" "${minion}"
   done

   IFS="${DEFAULT_IFS}"
}


work_clones()
{
   local reposdir="$1"
   local clones="$2"
   local required_clones="$3"
   local autoupdate="$4"

   local clone
   local name
   local url
   local branch
   local scm
   local tag
   local stashdir
   local dstdir

   local actionitems
   local remember
   local repotype
   local oldstashdir

   case "${reposdir}" in
      *embedded)
        repotype="embedded "
      ;;

      *)
        repotype=""
      ;;
   esac

   log_debug "Working \"${clones}\""

   IFS="
"
   for clone in ${clones}
   do
      IFS="${DEFAULT_IFS}"

      if [ -z "${clone}" ]
      then
         continue
      fi

      #
      # optimization, try not to redo fetches
      #
      echo "${__IGNORE__}" | fgrep -s -q -x "${clone}" > /dev/null
      if [ $? -eq 0 ]
      then
         continue
      fi

      log_debug "${C_INFO}Doing clone \"${clone}\" ..."

      __REFRESHED__="`add_line "${__REFRESHED__}" "${clone}"`"

      parse_raw_clone "${clone}"
      process_raw_clone

      stashdir="${dstdir}"
      [ -z "${stashdir}" ] && internal_fail "empty stashdir"

      case "${name}" in
         http:*|https:*)
            internal_fail "failed name: ${name}"
      esac

      case "${stashdir}" in
         http:*|https:*)
            internal_fail "failed stashdir: ${stashdir}"
      esac

      local skip

      skip="NO"

      if is_minion_bootstrap_project "${name}"
      then
         log_fluff "\"${name}\" is a minion, ignoring possible changes"
         clone="`echo "${name};${name};${branch};minion;${tag}" | sed 's/;*$//'`"
         remember="YES"
      else
         actionitems="`required_action_for_clone "${clone}" \
                                                 "${reposdir}" \
                                                 "${name}" \
                                                 "${url}" \
                                                 "${branch}" \
                                                 "${scm}" \
                                                 "${tag}" \
                                                 "${stashdir}"`" || exit 1

         log_debug "${C_INFO}Actions for \"${name}\": ${actionitems:-none}"

         IFS="
"
         for item in ${actionitems}
         do
            IFS="${DEFAULT_IFS}"

            remember="YES"

            case "${item}" in
               "checkout")
                  if ! checkout_repository "${reposdir}" \
                                           "${name}" \
                                           "${url}" \
                                           "${branch}" \
                                           "${scm}" \
                                           "${tag}" \
                                           "${stashdir}"
                  then
                     fail "Failed to checkout"
                  fi
               ;;

               "clone")
                  clone_repository "${reposdir}" \
                                   "${name}" \
                                   "${url}" \
                                   "${branch}" \
                                   "${scm}" \
                                   "${tag}" \
                                   "${stashdir}"

                  case "$?" in
                     1)
                        #
                        # check if clone is an optional install component
                        # it is if it doesn't show up in "required"
                        # required is built by mulle-bootstrap usually
                        # you specify optionals, by specifying the required
                        # clones and leaving the optional out
                        #
                        if ! echo "${required_clones}" | fgrep -s -q -x "${name}" > /dev/null
                        then
                           log_info "${C_MAGENTA}${C_BOLD}${name}${C_INFO} is missing, but it's not required."

                           merge_line_into_file "${REPOS_DIR}/.missing" "${name}"
                           skip="YES"
                           continue
                        fi

                        log_debug "don't continue, because a required fetch failed"
                        exit 1  # means exit
                     ;;

                     2)
                        # if we used a symlink, we want to memorize that
                        scm="symlink"
                     ;;
                  esac
               ;;

               "ignore")
                  remember="NO"
               ;;

               move*)
                  oldstashdir="${item:5}"

                  log_info "Moving ${repotype}stash ${C_MAGENTA}${C_BOLD}${name}${C_INFO} from \"${oldstashdir}\" to \"${stashdir}\""

                  if ! exekutor mv ${COPYMOVEFLAGS} "${oldstashdir}" "${stashdir}"  >&2
                  then
                     fail "Move failed!"
                  fi
               ;;

               "upgrade")
                  upgrade_repository "${reposdir}" \
                                     "${name}" \
                                     "${url}" \
                                     "${branch}" \
                                     "${scm}" \
                                     "${tag}" \
                                     "${stashdir}"
               ;;

               remove*)
                  oldstashdir="${item:7}"
                  if [ -z "${oldstashdir}" ]
                  then
                     oldstashdir="`get_old_stashdir "${reposdir}" "${name}"`"
                  fi

                  bury_stash "${reposdir}" "${name}" "${oldstashdir}"
               ;;

               "set-remote")
                  log_info "Changing ${repotype}remote to \"${url}\""

                  local remote

                  remote="`git_get_default_remote "${stashdir}"`"
                  if [ -z "${remote}" ]
                  then
                     fail "Could not figure out a remote for \"$PWD/${stashdir}\""
                  fi
                  git_set_url "${stashdir}" "${remote}" "${url}"
               ;;

               *)
                  internal_fail "Unknown action item \"${item}\""
               ;;
            esac
         done

         if [ "${autoupdate}" = "YES" ]
         then
            bootstrap_auto_update "${name}" "${stashdir}"
         fi

         # create clone as it is now
         clone="`echo "${url};${dstdir};${branch};${scm};${tag}" | sed 's/;*$//'`"
      fi

      if [ "${skip}" = "YES" ]
      then
         log_debug "skipping to next clone because..."
         continue
      fi

      #
      # always remember, what we have now (except if its a minion)
      #
      if [ "${remember}" = "YES" ]
      then
         # branch could be overwritten
         log_debug "${C_INFO}Remembering ${clone} ..."

         remember_stash_of_repository "${clone}" \
                                      "${reposdir}" \
                                      "${name}"  \
                                      "${stashdir}" \
                                      "${PARENT_CLONE}"
      else
         log_debug "ignoring because..."
      fi
      mark_stash_as_alive "${reposdir}" "${name}"
   done

   IFS="${DEFAULT_IFS}"
}

#
#
#
fetch_once_embedded_repositories()
{
   log_debug "fetch_once_embedded_repositories"

   (
      STASHES_DEFAULT_DIR=""
      STASHES_ROOT_DIR=""
      OPTION_ALLOW_CREATING_SYMLINKS="${OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS}" ;

      local clones
      local required_clones

      clones="`read_root_setting "embedded_repositories"`" ;
      required_clones="`read_root_setting "embedded_required"`" ;
      work_clones "${EMBEDDED_REPOS_DIR}" "${clones}" "${required_clones}" "NO"
   ) || exit 1
}


#
#
#
fetch_once_minions_embedded_repositories()
{
   log_debug "fetch_once_minions_embedded_repositories"

   local minions

   minions="`read_root_setting minions`"

   local minion

   IFS="
"
   for minion in ${minions}
   do
      IFS="${DEFAULT_IFS}"

      if [ -z "${minion}" ]
      then
         continue
      fi

      log_debug "${C_INFO}Doing minion \"${minion}\" ..."

      local autodir

      autodir="${BOOTSTRAP_DIR}.auto/.deep/${minion}.d"

      if [ ! -d "${autodir}" ]
      then
         log_fluff "${autodir} not present, so done"
         continue
      fi

      local autodir

      reposdir="${REPOS_DIR}/.deep/${minion}.d"

      (
         STASHES_DEFAULT_DIR=""
         STASHES_ROOT_DIR=""
         OPTION_ALLOW_CREATING_SYMLINKS="${OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS}" ;

         local clones
         local required_clones

         clones="`read_setting "${autodir}/embedded_repositories"`" ;
         required_clones="`read_setting "${autodir}/embedded_required"`" ;
         work_clones "${reposdir}" "${clones}" "${required_clones}" "NO"
      ) || exit 1
   done

   IFS="${DEFAULT_IFS}"
}


_fetch_once_deep_embedded_repository()
{
   local reposdir="$1"  # ususally .bootstrap.repos
   local name="$2"      # name of the clone
   local url="$3"       # URL of the clone
   local branch="$4"    # branch of the clone
   local scm="$5"       # scm to use for this clone
   local tag="$6"       # tag to checkout of the clone
   local stashdir="$7"  # stashdir of this clone (absolute or relative to $PWD)

   local autodir

   autodir="${BOOTSTRAP_DIR}.auto/.deep/${name}.d"

   if [ ! -d "${autodir}" ]
   then
      log_fluff "${autodir} not present, so done"
      return
   fi

   reposdir="${reposdir}/.deep/${name}.d"
   (
      STASHES_DEFAULT_DIR=""
      STASHES_ROOT_DIR="${stashdir}"  # hackish
      OPTION_ALLOW_CREATING_SYMLINKS="${OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS}" ;
      PARENT_REPOSITORY_NAME="${name}"
      PARENT_CLONE="${clone}"

      local clones
      local required_clones

      # ugliness
      clones="`read_setting "${autodir}/embedded_repositories"`" ;
      required_clones="`read_setting "${autodir}/embedded_required"`" ;
      work_clones "${reposdir}" "${clones}" "${required_clones}" "NO"
   ) || exit 1
}


# but not minions
fetch_once_deep_embedded_repositories()
{
   log_debug "fetch_once_deep_embedded_repositories"

   local permissions

   walk_auto_repositories "repositories"  \
                          "_fetch_once_deep_embedded_repository" \
                          "${permissions}" \
                          "${REPOS_DIR}"
}


extract_minion_precis()
{
   local minions

   minions="`read_root_setting minions`"
   auto_update_minions "${minions}"
}


fetch_loop_repositories()
{
   local loops
   local before
   local after
   local auxbefore
   local auxafter
   local required

   log_debug "fetch_loop_repositories"

   loops=""
   before=""
   auxbefore=""

   __IGNORE__=""

   while :
   do
      loops="${loops}X"
      case "${loops}" in
         XXXXXXXXXXXXXXXX)
            internal_fail "Loop overflow in worker loop"
         ;;
      esac

      after="${before}"
      before="`read_root_setting "repositories" | sort`"
      auxafter="${auxbefore}"
      auxbefore="`read_root_setting "additional_repositories" | sort`"

      if [ "${after}" = "${before}" -a "${auxafter}" = "${auxbefore}" ]
      then
         log_fluff "Repositories files are unchanged, so done"
         break
      fi

      if [ -z "${__IGNORE__}" ]
      then
         log_fluff "Get back in the ring to take another swing"
      fi

      __REFRESHED__=""

      required="`read_root_setting "required"`"
      work_clones "${REPOS_DIR}" "${before}" "${required}" "YES"
      work_clones "${REPOS_DIR}" "${auxbefore}" "${required}" "YES"

      __IGNORE__="`add_line "${__IGNORE__}" "${__REFRESHED__}"`"

   done
}

                      #----#
### Main fetch loop   #    #    #----#
                      #----#    #----#    #----#

assume_stashes_are_zombies()
{
   if [ -d "${REPOS_DIR}" ]
   then
      zombify_embedded_repository_stashes
      [ "${OPTION_EMBEDDED_ONLY}" = "YES" ] && return

      zombify_repository_stashes
      zombify_deep_embedded_repository_stashes
   fi
}


bury_zombies_in_graveyard()
{
   if [ -d "${REPOS_DIR}" ]
   then
      bury_embedded_repository_zombies
      [ "${OPTION_EMBEDDED_ONLY}" = "YES" ] && return

      bury_repository_zombies
      bury_deep_embedded_repository_zombies
   fi
}


#
# the main fetch loop as documented somewhere with graphviz
#
fetch_loop()
{
   unpostpone_trace

   # this is wrong, we just do "stashes" though
   assume_stashes_are_zombies

   bootstrap_auto_create

   if is_master_bootstrap_project
   then
     log_info "Extracting minions' precis ..."

      extract_minion_precis

      fetch_once_minions_embedded_repositories
   fi

   log_info "Checking repositories ..."

   fetch_once_embedded_repositories

   if [ "${OPTION_EMBEDDED_ONLY}" = "NO" ]
   then
      fetch_loop_repositories

      fetch_once_deep_embedded_repositories || exit 1
   fi

   bootstrap_auto_final

   bury_zombies_in_graveyard
}

#
# the three commands
#
_common_fetch()
{

   fetch_loop "${REPOS_DIR}"

   #
   # do this afterwards, because brews will have been composited
   # now
   #
   case "${BREW_PERMISSIONS}" in
      fetch|update|upgrade)
         brew_install_brews
      ;;
   esac

   check_tars
}


_common_update()
{
   case "${BREW_PERMISSIONS}" in
      update|upgrade)
         brew_update_main
      ;;
   esac

   update_embedded_repositories
   [ "${OPTION_EMBEDDED_ONLY}" = "YES" ] && return

   update_repositories "$@"
   update_deep_embedded_repositories
}


_common_upgrade()
{
   case "${BREW_PERMISSIONS}" in
      upgrade)
         brew_upgrade_main
      ;;
   esac

   upgrade_embedded_repositories
   [ "${OPTION_EMBEDDED_ONLY}" = "YES" ] && return

   upgrade_repositories "$@"
   upgrade_deep_embedded_repositories

   _common_fetch  # update what needs to be update
}


_common_main()
{
   [ -z "${MULLE_BOOTSTRAP_REPOSITORIES_SH}" ]      && . mulle-bootstrap-repositories.sh
   [ -z "${MULLE_BOOTSTRAP_LOCAL_ENVIRONMENT_SH}" ] && . mulle-bootstrap-local-environment.sh
   [ -z "${MULLE_BOOTSTRAP_SETTINGS_SH}" ]          && . mulle-bootstrap-settings.sh

   local OPTION_CHECK_USR_LOCAL_INCLUDE="NO"
   local MULLE_FLAG_FOLLOW_SYMLINKS="NO"
   local OPTION_ALLOW_CREATING_SYMLINKS="NO"
   local OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS="NO"
   local OPTION_ALLOW_SEARCH_CACHES="YES"
   local OPTION_EMBEDDED_ONLY="NO"
   local OVERRIDE_BRANCH
   local DONT_WARN_SCRIPTS="NO"

   local ROOT_DIR="`pwd -P`"

   OPTION_CHECK_USR_LOCAL_INCLUDE="`read_config_setting "check_usr_local_include" "NO"`"
   OVERRIDE_BRANCH="`read_config_setting "override_branch"`"


   DONT_WARN_SCRIPTS="`read_config_setting "dont_warn_scripts" "${MULLE_FLAG_ANSWER:-NO}"`"

   if [ "${DONT_WARN_SCRIPTS}" = "YES" ]
   then
      log_verbose "Script checking disabled"
   fi

   case "${UNAME}" in
      mingw)
      ;;

      *)
         OPTION_ALLOW_CREATING_SYMLINKS="`read_config_setting "symlinks" "${MULLE_FLAG_ANSWER}"`"
         OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS="`read_config_setting "embedded_symlinks" "NO"`"
      ;;
   esac

   #
   # it is useful, that fetch understands build options and
   # ignores them
   #
   while [ $# -ne 0 ]
   do
      case "$1" in
         -h|-help|--help)
            ${USAGE}
         ;;

         -cu|--check-usr-local|--check-usr-local)
            OPTION_CHECK_USR_LOCAL_INCLUDE="YES"
         ;;

         -e|--embedded-only)
            OPTION_EMBEDDED_ONLY="YES"
         ;;

         # update symlinks, dangerous (duplicate to flags for laziness reasons)
         --follow-symlinks)
            MULLE_FLAG_FOLLOW_SYMLINKS="YES"
         ;;

         # allow creating symlinks instead of clones for repositories
         -l|--symlink-creation|--symlinks)
            OPTION_ALLOW_CREATING_SYMLINKS="YES"
         ;;

         #
         # create symlinks instead of clones for embedded_repositories
         # and deep embedded_repositories. If the clones are symlinked
         # this will be ignored, except if follow-symlinks is
         # active. --follow-symlinks is really not safe!
         #
         --embedded-symlink-creation|--embedded-symlinks)
            OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS="YES"
         ;;

         --no-caches)
            OPTION_ALLOW_SEARCH_CACHES="NO"
         ;;

         --no-symlink-creation|--no-symlinks)
            MULLE_FLAG_FOLLOW_SYMLINKS="NO"
            OPTION_ALLOW_CREATING_SYMLINKS="NO"
            OPTION_ALLOW_CREATING_EMBEDDED_SYMLINKS="NO"
         ;;

         # TODO: outdated!
         # build options with no parameters
         -K|--clean|-k|--no-clean|--use-prefix-libraries|--debug|--release)
            if [ -z "${MULLE_BOOTSTRAP_WILL_BUILD}" ]
            then
               log_error "${MULLE_EXECUTABLE_FAIL_PREFIX}: Unknown fetch option $1"
               ${USAGE}
            fi
         ;;

         # build options with one parameter
         # TODO: outdated!
         -j|--cores|-c|--configuration|--prefix)
            if [ -z "${MULLE_BOOTSTRAP_WILL_BUILD}" ]
            then
               log_error "${MULLE_EXECUTABLE_FAIL_PREFIX}: Unknown fetch option $1"
               ${USAGE}
            fi

            if [ $# -eq 1 ]
            then
               log_error "${MULLE_EXECUTABLE_FAIL_PREFIX}: Missing parameter to fetch option $1"
               ${USAGE}
            fi
            shift
         ;;

         -*)
            log_error "${MULLE_EXECUTABLE_FAIL_PREFIX}: Unknown fetch option $1"
            ${USAGE}
         ;;

         *)
            break
         ;;
      esac

      shift
   done

   [ -z "${MULLE_BOOTSTRAP_AUTO_UPDATE_SH}" ]     && . mulle-bootstrap-auto-update.sh
   [ -z "${MULLE_BOOTSTRAP_COMMON_SETTINGS_SH}" ] && . mulle-bootstrap-common-settings.sh
   [ -z "${MULLE_BOOTSTRAP_SCM_SH}" ]             && . mulle-bootstrap-scm.sh
   [ -z "${MULLE_BOOTSTRAP_SCRIPTS_SH}" ]         && . mulle-bootstrap-scripts.sh
   [ -z "${MULLE_BOOTSTRAP_ZOMBIFY_SH}" ]         && . mulle-bootstrap-zombify.sh

   #
   # should we check for '/usr/local/include/<name>' and don't fetch if
   # present (somewhat dangerous, because we do not check versions)
   #

   if [ "${COMMAND}" = "fetch" ]
   then
      if [ $# -ne 0 ]
      then
         log_error "Additional parameters not allowed for fetch (" "$@" ")"
         ${USAGE}
      fi
   fi

   #
   # Run prepare scripts if present
   #
   case "${COMMAND}" in
      update|upgrade)
         if dir_is_empty "${REPOS_DIR}"
         then
            log_info "Nothing to update, fetch first"

            return 0
         fi
      ;;
   esac


   local default_permissions

   #
   # possible values none|fetch|update|upgrade
   # the local scheme with addictions really works
   # best on darwin, linux can't use bottles locally
   #
   default_permissions="none"
   case "${UNAME}" in
      darwin|linux)
         default_permissions="upgrade"
      ;;
   esac

   BREW_PERMISSIONS="`read_config_setting "brew_permissions" "${default_permissions}"`"
   case "${BREW_PERMISSIONS}" in
      none|fetch|update|upgrade)
      ;;

      *)
        fail "brew_permissions must be either: none|fetch|update|upgrade)"
      ;;
   esac

   remove_file_if_present "${REPOS_DIR}/.build_done"
   remove_file_if_present "${REPOS_DIR}/.fetch_done"
   create_file_if_missing "${REPOS_DIR}/.fetch_started"

   if [ "${BREW_PERMISSIONS}" != "none" ]
   then
      [ -z "${MULLE_BOOTSTRAP_BREW_SH}" ] && . mulle-bootstrap-brew.sh
   fi

   [ -z "${DEFAULT_IFS}" ] && internal_fail "IFS fail"

   case "${COMMAND}" in
      update)
         _common_update "$@"
      ;;

      upgrade)
         _common_upgrade "$@"
      ;;
   esac

   _common_fetch "$@"

   remove_file_if_present "${REPOS_DIR}/.fetch_started"

   #
   # only say we are done if a build_order was created
   #
   if [ -f "${BOOTSTRAP_DIR}.auto/build_order" ]
   then
      create_file_if_missing "${REPOS_DIR}/.fetch_done"
   fi

   if read_yes_no_config_setting "upgrade_gitignore" "YES"
   then
      if [ -d .git ]
      then
         append_dir_to_gitignore_if_needed "${BOOTSTRAP_DIR}.auto"
         append_dir_to_gitignore_if_needed "${BOOTSTRAP_DIR}.local"
         append_dir_to_gitignore_if_needed "${DEPENDENCIES_DIR}"
         if [ "${brew_permissions}" != "none" ]
         then
            append_dir_to_gitignore_if_needed "${ADDICTIONS_DIR}"
         fi
         append_dir_to_gitignore_if_needed "${REPOS_DIR}"
         if [ ! -z "${STASHES_DEFAULT_DIR}" -a -d "${STASHES_DEFAULT_DIR}"  ]
         then
            append_dir_to_gitignore_if_needed "${STASHES_DEFAULT_DIR}"
         fi
      fi
   fi
}


fetch_main()
{
   log_debug "::: fetch begin :::"

   USAGE="fetch_usage"
   COMMAND="fetch"
   _common_main "$@"

   log_debug "::: fetch end :::"
}


update_main()
{
   log_debug "::: update begin :::"

   USAGE="fetch_usage"
   COMMAND="update"
   _common_main "$@"

   log_debug "::: update end :::"
}


upgrade_main()
{
   log_debug "::: upgrade begin :::"

   USAGE="fetch_usage"
   COMMAND="upgrade"
   _common_main "$@"

   log_debug "::: upgrade end :::"
}