Get best of the week

Migration de Gitolite vers GitLab avec un script Shell

Le processus de migration est souvent une tâche difficile, surtout lorsque la quantité d'informations à transférer est si importante qu'il devient plus rentable de l'automatiser. C'est la nécessité de migrer de Gitolite vers GitLab qui m'a incité à écrire un article sur mon expérience en la matière.


Pour la migration des référentiels, je vais utiliser un ordinateur avec le système d'exploitation CentOS 7. Sur celui-ci, je dois installer le jeu d'applications suivant: git, ssh-agent, curl, jq, xargs .


Pour commencer, nous devons obtenir la clé d'accès à l'API GitLab. Dans l'interface Web, accédez à la section des paramètres utilisateur. Ensuite, sélectionnez l'élément de menu «Jetons d'accès» . Dans le formulaire qui s'ouvre, vous devez spécifier le nom de la clé reçue et vous pouvez également spécifier la période pendant laquelle la clé sera active. Ci-dessous, cochez la case à côté de l'élément api et cliquez sur le bouton "Créer un jeton d'accès personnel" . Cette clé sera utilisée lors de l'envoi de demandes à l'API GitLab.


  1. Préparer l'environnement de travail


    GitLab SSH, , GitLab. GitLab «Admin area» «Deploy keys» «New deploy key». , . , GitLab. bash:


    curl -k -X GET --header "PRIVATE-TOKEN: <your private token>" <gitlab url>/api/v4/deploy_keys

    : SSH git , ssh-agent. :


    eval `ssh-agent`

    ssh-add ~/.ssh/id_rsa

  2. GitLab API


    GitLab, POST API GitLab, , . Projects API GitLab GitLab.


    # $1 - gitlab remote url
# $2 - api access token
# $3 - remote repository name
# $4 - remote repository namespace identifier
initRemoteRepository() {
    local response=`curl -s -k -X POST --header "Private-Token: $2" --data "name=$3&namespace_id=$4&visibility=private&description=$3" $1/api/v4/projects`
    local id=`echo $response | jq -r ".id"`
    echo "$id"
}

    initRemoteRepository . .
    : –k , GitLab .


  3. API


    , , . Deploy Keys API GitLab GitLab.


    # $1 - remote url
# $2 - api token
# $3 - repository identifier
# $4 - deploy key identifier
enableRepositoryDeployKey() {
    local response=`curl -s -k -X POST --header "Private-Token: $2" $1/api/v4/projects/$3/deploy_keys/$4/enable`
    echo "$response"
}

    enableRepositoryDeployKey .


    # $1 - remote url
# $2 - api token
# $3 - repository identifier
# $4 - deploy key identifier
setRepositoryWriteAccess() {
    local response=`curl -s -k -X PUT --header "PRIVATE-TOKEN: $2" --header "Content-Type: application/json" --data '{"can_push": true}' $1/api/v4/projects/$3/deploy_keys/$4`
    echo "$response"
}

    setRepositoryWriteAccess .



  4. , . GitLab, . . GitLab.


    # $1 - branch
# $2 – separator
getCanonicalBranchName() {
    local branch=""
    IFS=$2 read -r -a array <<< "$1"
    length=${#array[@]}
    for index in "${!array[@]}"
    do
        if [ $index -gt 0 ]; then
            if [ $index -eq 1 ]; then
                local branch="${array[index]}"
            else
                local branch="$branch/${array[index]}"
            fi
        fi
    done
    echo "$branch"
}
PSWD="$(dirname "$0")"
cd $PSWD
while IFS= read -r REPOSITORY
do
    echo "====================  <$REPOSITORY>  ===================="
    GITOLITE_REPO=$GITOLITE$REPOSITORY
    echo "Run cloning remote repository $GITOLITE_REPO"
    git clone $GITOLITE_REPO
    cd $REPOSITORY
    GITLAB_REPO=$GITLAB$REPOSITORY.git
    echo "Initialize remote gitgab repository $GITLAB_REPO"
    PROJECT=$(initRemoteRepository $GITLAB_URL $API_TOKEN $REPOSITORY)
    echo "Remote repository initialized identifier $PROJECT"
    echo "Add deploy key and enable write access to remote repository $REPOSITORY ($PROJECT)"
    response=$(enableRepositoryDeployKey $GITLAB_URL $API_TOKEN $PROJECT $DEPLOY_KEY_ID)
    response =$(setRepositoryWriteAccess $GITLAB_URL $API_TOKEN $PROJECT $DEPLOY_KEY_ID)
    echo "Add new remote url for repository $GITLAB_REPO"
    git remote add gitlab $GITLAB_REPO
    # push all branches
    IFS_BACK=$IFS
    IFS=$'\n'
    branches=$(git branch -r)
    for branchName in $branches; do
        trimBranchName=`echo $branchName | xargs`
        canonicalBranchName=$(getCanonicalBranchName $trimBranchName '/')
        echo "$trimBranchName ($canonicalBranchName) init push"
        git checkout -b $canonicalBranchName remotes/origin/$canonicalBranchName
        git push -f gitlab $canonicalBranchName
    done
    IFS=$IFS_BACK
    cd ..
    rm -r -f $REPOSITORY
    echo "====================  <$REPOSITORY>  ===================="
    echo ""
done < "$REPOSITORIES"

    , . .



Script complet
REPOSITORIES="<repositories_file>"
GITOLITE="<gitolite_ssh_url>"
GITLAB_URL="<gitlab_api_url>"
GITLAB="<gitolite_ssh_url>"
API_TOKEN="<api_access_token>"
DEPLOY_KEY_ID="<deploy_key_identifier>"
# $1 - gitlab remote url
# $2 - api access token
# $3 - remote repository name
# $4 - remote repository namespace identifier
initRemoteRepository() {
        local response=`curl -s -k -X POST --header "Private-Token: $2" --data "name=$3&namespace_id=$4&visibility=private&description=$3" $1/api/v4/projects`
        local id=`echo $response | jq -r ".id"`
        echo "$id"
}
# $1 - remote url
# $2 - api token
# $3 - repository identifier
# $4 - deploy key identifier
setRepositoryWriteAccess() {
    local response=`curl -s -k -X PUT --header "PRIVATE-TOKEN: $2" --header "Content-Type: application/json" --data '{"can_push": true}' $1/api/v4/projects/$3/deploy_keys/$4`
    echo "$response"
}
# $1 - remote url
# $2 - api token
# $3 - repository identifier
# $4 - deploy key identifier
enableRepositoryDeployKey() {
    local response=`curl -s -k -X POST --header "Private-Token: $2" $1/api/v4/projects/$3/deploy_keys/$4/enable`
    echo "$response"
}
# $1 - branch
# $2 – separator
getCanonicalBranchName() {
    local branch=""
    IFS=$2 read -r -a array <<< "$1"
    length=${#array[@]}
    for index in "${!array[@]}"
    do
        if [ $index -gt 0 ]; then
            if [ $index -eq 1 ]; then
                local branch="${array[index]}"
            else
                local branch="$branch/${array[index]}"
            fi
        fi
    done
    echo "$branch"
}
PSWD="$(dirname "$0")"
cd $PSWD
while IFS= read -r REPOSITORY
do
    echo "====================  <$REPOSITORY>  ===================="
    GITOLITE_REPO=$GITOLITE$REPOSITORY
    echo "Run cloning remote repository $GITOLITE_REPO"
    git clone $GITOLITE_REPO
    cd $REPOSITORY
    GITLAB_REPO=$GITLAB$REPOSITORY.git
    echo "Initialize remote gitgab repository $GITLAB_REPO"
    PROJECT=$(initRemoteRepository $GITLAB_URL $API_TOKEN $REPOSITORY)
    echo "Remote repository initialized identifier $PROJECT"
    echo "Add deploy key and enable write access to remote repository $REPOSITORY ($PROJECT)"
    response=$(enableRepositoryDeployKey $GITLAB_URL $API_TOKEN $PROJECT $DEPLOY_KEY_ID)
    response=$(setRepositoryWriteAccess $GITLAB_URL $API_TOKEN $PROJECT $DEPLOY_KEY_ID)
    echo "Add new remote url for repository $GITLAB_REPO"
    git remote add gitlab $GITLAB_REPO
    # push all branches
    IFS_BACK=$IFS
    IFS=$'\n'
    branches=$(git branch -r)
    for branchName in $branches; do
        trimBranchName=`echo $branchName | xargs`
        canonicalBranchName=$(getCanonicalBranchName $trimBranchName '/')
        echo "$trimBranchName ($canonicalBranchName) init push"
        git checkout -b $canonicalBranchName remotes/origin/$canonicalBranchName
        git push -f gitlab $canonicalBranchName
    done
    IFS=$IFS_BACK
    cd ..
    rm -r -f $REPOSITORY
    echo "====================  <$REPOSITORY>  ===================="
    echo ""
done < "$REPOSITORIES"

Exemple de sortie de console
====================  <project>  ====================
Run cloning remote repository git@skynet-uro.bank.srv:project
Cloning into 'project'...
remote: Counting objects: 62, done.
remote: Compressing objects: 100% (61/61), done.
remote: Total 62 (delta 21), reused 0 (delta 0)
Receiving objects: 100% (62/62), 15.57 KiB | 0 bytes/s, done.
Resolving deltas: 100% (21/21), done.
Current folder path: /app/migration/project
Initialize remote gitgab repository git@127.0.0.1:project.git
Remote repository initialized identifier 222
Enable deploy key write access to remote repository project (222)
Set write access to remote repository project (222)
Add new remote url for repository git@127.0.0.1:project.git
Push to new remote gitlab repository git@127.0.0.1:project.git
Counting objects: 55, done.
Delta compression using up to 2 threads.
Compressing objects: 100% (38/38), done.
Writing objects: 100% (55/55), 14.65 KiB | 0 bytes/s, done.
Total 55 (delta 18), reused 50 (delta 16)
To git@127.0.0.1:project.git
 * [new branch]      master -> master
<--- gitlab/master (master) init push
fatal: A branch named 'master' already exists.
Everything up-to-date
---> gitlab/master (master) success pushed
<--- origin/v0.0.0 (v0.0.0) init push
Branch v0.0.0 set up to track remote branch v0.0.0 from origin.
Switched to a new branch 'v0.0.0'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.0, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.0
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.0 -> v0.0.0
---> origin/v0.0.0 (v0.0.0) success pushed
<--- origin/v0.0.0-13 (v0.0.0-13) init push
Branch v0.0.0-13 set up to track remote branch v0.0.0-13 from origin.
Switched to a new branch 'v0.0.0-13'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.0-13, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.0-13
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.0-13 -> v0.0.0-13
---> origin/v0.0.0-13 (v0.0.0-13) success pushed
<--- origin/v0.0.0-15 (v0.0.0-15) init push
Branch v0.0.0-15 set up to track remote branch v0.0.0-15 from origin.
Switched to a new branch 'v0.0.0-15'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.0-15, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.0-15
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.0-15 -> v0.0.0-15
---> origin/v0.0.0-15 (v0.0.0-15) success pushed
<--- origin/v0.0.1 (v0.0.1) init push
Branch v0.0.1 set up to track remote branch v0.0.1 from origin.
Switched to a new branch 'v0.0.1'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.1, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.1
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.1 -> v0.0.1
---> origin/v0.0.1 (v0.0.1) success pushed
<--- origin/v0.0.1-11 (v0.0.1-11) init push
Branch v0.0.1-11 set up to track remote branch v0.0.1-11 from origin.
Switched to a new branch 'v0.0.1-11'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.1-11, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.1-11
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.1-11 -> v0.0.1-11
---> origin/v0.0.1-11 (v0.0.1-11) success pushed
<--- origin/v0.0.2 (v0.0.2) init push
Branch v0.0.2 set up to track remote branch v0.0.2 from origin.
Switched to a new branch 'v0.0.2'
Total 0 (delta 0), reused 0 (delta 0)
remote:
remote: To create a merge request for v0.0.2, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.2
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.2 -> v0.0.2
---> origin/v0.0.2 (v0.0.2) success pushed
<--- origin/v0.0.3 (v0.0.3) init push
Branch v0.0.3 set up to track remote branch v0.0.3 from origin.
Switched to a new branch 'v0.0.3'
Counting objects: 7, done.
Delta compression using up to 2 threads.
Compressing objects: 100% (4/4), done.
Writing objects: 100% (4/4), 592 bytes | 0 bytes/s, done.
Total 4 (delta 3), reused 1 (delta 0)
remote:
remote: To create a merge request for v0.0.3, visit:
remote:   https://127.0.0.1:8082/project/merge_requests/new?merge_request%5Bsource_branch%5D=v0.0.3
remote:
To git@127.0.0.1:project.git
 * [new branch]      v0.0.3 -> v0.0.3
---> origin/v0.0.3 (v0.0.3) success pushed
====================  <project>  ====================

More articles:


All Articles