From 4667b5de579c96220334ac1f2253cfa26781193e Mon Sep 17 00:00:00 2001 From: Thomas Maurice Date: Sun, 24 Mar 2024 12:46:12 +0100 Subject: [PATCH] upgrade(README): Improve README, clarify instructions and add documentation links --- .gitea/workflows/on_pr.yml | 5 ++ .gitea/workflows/on_push_master.yml | 5 ++ Makefile | 2 +- README.md | 74 ++++++++++++++++++++++++----- 4 files changed, 72 insertions(+), 14 deletions(-) diff --git a/.gitea/workflows/on_pr.yml b/.gitea/workflows/on_pr.yml index 9424d0c..8e8ec77 100644 --- a/.gitea/workflows/on_pr.yml +++ b/.gitea/workflows/on_pr.yml @@ -16,13 +16,18 @@ jobs: python-version: '3.10.13' - name: "Creates virtual environment" run: make venv + # Loads up the virtualenv - name: Activate virtualenv run: | source venv/bin/activate echo PATH=$PATH >> $GITHUB_ENV + # Installs playbooks we are using - name: Install galaxy requirements run: ansible-galaxy install -r requirements.yml + # Checks for yaml correctness - name: "Run YAML linting" run: yamllint -c .yamllint.yml . + # Runs an *ansible specific* linter to check for mistakes + # and or/bad patterns - name: "Run Ansible linting" run: ansible-lint diff --git a/.gitea/workflows/on_push_master.yml b/.gitea/workflows/on_push_master.yml index 76fb2f4..27f19ff 100644 --- a/.gitea/workflows/on_push_master.yml +++ b/.gitea/workflows/on_push_master.yml @@ -18,13 +18,18 @@ jobs: python-version: '3.10.13' - name: "Creates virtual environment" run: make venv + # Loads up the virtualenv - name: Activate virtualenv run: | source venv/bin/activate echo PATH=$PATH >> $GITHUB_ENV + # Installs playbooks we are using - name: Install galaxy requirements run: ansible-galaxy install -r requirements.yml + # Checks for yaml correctness - name: "Run YAML linting" run: yamllint -c .yamllint.yml . + # Runs an *ansible specific* linter to check for mistakes + # and or/bad patterns - name: "Run Ansible linting" run: ansible-lint diff --git a/Makefile b/Makefile index 9227d03..46d90f4 100644 --- a/Makefile +++ b/Makefile @@ -6,4 +6,4 @@ install: ansible-galaxy install -r requirements.yml action: - act -W .gitea/workflows/on_pr.yml \ No newline at end of file + act -W .gitea/workflows/on_pr.yml diff --git a/README.md b/README.md index d3c5917..2fdeb97 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,42 @@ -# ansible-polytech-2023 +# ansible-polytech-2024 This is the repo that will serve as the support for the ASR practical work session of january. You should probably work in this directory and add code to it, it is going to be easier than creating a new ansible repository from scratch. You are also welcome to commit to this repository to checkpoint your work, as well as push it into any repository that polytech gives you access to. -The goal here is to deploy a Synapse Matrix server, an opensource decentralised messageng app, using industry standard tools such as ansible and docker. +The goal here is to deploy a Synapse Matrix server, an opensource decentralised messaging app, using industry standard tools such as ansible and docker. By the end of this lab you'll be able to message each other in an end to end encrypted fashion using your own servers! +## How it's going to work +* Log into https://gitea.plil.fr with your school credentials +* Then _fork_ [this repository](https://gitea.plil.fr/tmaurice/polytech-ansible) (you need to do that only once per group) +* Modify the repo's code as you see fit and remember to *commit and push often*. This is how Xavier and myself are going to assess progress and be able to grade your work. + +Optionally I would recomment you to add this repository as a remote so you can pull fixes should I have to push emergency fixes to the instructions (it happens more or less every year): + +``` +$ git remote add upstream https://git.plil.fr/tmaurice/polytech-ansible +# If you want to get the latest code merged with your branch +$ git pull upstream master +``` + +:warning: Please use explicit and descriptive commit messages when committing to git. + +### Optional setup: a gitea runner for continous integration + +You can if you so wish deploy a [gitea runner](https://gitea.com/gitea/act_runner) somewhere, preferably on your zabeth. I would recommend you set the runner up for your user, and enable the "actions" in the settings of your newly created repo. + +This way every time you push your code, the checks in `.gitea/workflows` will be ran and you will have a live preview of the state of your code (how is the linting doing, what is the code quality, etc etc). + +If the consensus is "fuck that ain't nobody got time for that" I can also deploy one from my own server somewhere and you'll be able to use the runner for the entire length of the lab. + +:warning: you can run your gitea actions locally by running `make action` after installing the [`act` binary](https://github.com/nektos/act) + +The default provided CI pipelines only check for YAML and Ansible correctness, but you could imagine workflows where after these tests are run and are successful, we trigger an ansible run directly from the CI and the runner itself. Feel free to try :) + ## Setup +:warning: This setup has to be done on your Zabeth host. + You will need to install a few things to get started, buckle up. Firstly you need to export some environment variables in your working terminal to be able to talk to the outside world using the school's proxy: @@ -28,7 +57,7 @@ To create the virtualenv you need to run the following: ``` $ python3 -m venv ~/.ansible-venv -# then you want to "activate" the venv, you will need to do this for every new term you open +# then you want to "activate" the venv, you will need to do this for every new terminal you open $ . ~/.ansible-venv/bin/activate ``` @@ -36,7 +65,7 @@ $ . ~/.ansible-venv/bin/activate ### Install ansible -Install ansible via pip _after entering the venv_ +Install ansible via pip _after entering the virtualenv_ ``` $ pip install -r requirements.txt @@ -66,7 +95,9 @@ $ ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -P '' :warning: In real life, don't use `-P ''` because it creates your SSH key without a passphrase, it is ok for this lab, not for real life. -Lastly, look into `group_vars/all.yml`, go at the end of the file and add the created _public_ key in the root_user.default_root_keys (from ~/.ssh/id_ed25519.pub, or any other keys you created before hand). +:warning: Never, ever, EVER commit an unencrypted SSH keys to a git repository. If you need to add the key to the repository for some reason I would recommend you look up something like [sops](https://github.com/getsops/sops) to store an encrypted version of the secret in the repository. + +Lastly, look into `group_vars/all.yml`, go at the end of the file and add the created _public_ key in the root_user.default_root_keys (from `~/.ssh/id_ed25519.pub`, or any other keys you created before hand). ### One more thing, update your inventory @@ -109,9 +140,9 @@ Good, now you are good to go ! To deploy Synapse and Mastodon, you need to deploy a database server. We are going to use Postgres in this lab. You will for this need to use the `community.postgres_db` module for this. The `community.*` modules are modules written by the community and available to everyone, you will encounter similar modules when you will want to start deploying docker container! -For more details to do this, I refer you to [this link](https://stribny.name/blog/ansible-postgresql/) which will walk you through how to deal with this. +For more details to do this, I refer you to [the official ansible postgres documentation](https://docs.ansible.com/ansible/latest/collections/community/postgresql/index.html). -Note that you won't be using the `dnf` instruction here since DNF is the package manager for CentOS based systems, use `apt` instead. Similarily the packages are named a bit differently, you need to install (with ansible !) +Note that you will need to install (with ansible !) - `postgresql` - `python3-psycopg2` @@ -127,10 +158,23 @@ On the `pg_hba.conf` file add a line like that at the end: host all all scram-sha-256 ``` +You can (and should) use the [postgresql_pg_hba](https://docs.ansible.com/ansible/latest/collections/community/postgresql/postgresql_pg_hba_module.html#ansible-collections-community-postgresql-postgresql-pg-hba-module) module for that. This module controles how clients can connect/authorise themselves to postgres. + The netmask should look like `A.B.C.D/E` like `0.0.0.0/0` for everyone, for example. Also change the `listen_addresses` instruction to `'*'` in `postgresql.conf`. +Then create the [user](https://docs.ansible.com/ansible/latest/collections/community/postgresql/postgresql_user_module.html#ansible-collections-community-postgresql-postgresql-user-module) and the [database](https://docs.ansible.com/ansible/latest/collections/community/postgresql/postgresql_db_module.html#ansible-collections-community-postgresql-postgresql-db-module). Please note that you need to pass the following parameters to the database you are creating, otherwise Synapse will most likely complain about encoding issues and refuse to start: +```yaml + encoding: "UTF-8" + lc_collate: "C" + lc_ctype: "C" +``` + +Bonus point if you store the username in an encrypted fashion in your repository with either [ansible vault](https://docs.ansible.com/ansible/latest/vault_guide/vault_managing_passwords.html) or [sops](https://github.com/getsops/sops). + +If you end up using sops, please also encrypt your secret with the following age public key so I am able to decrypt them later on `age18rkuwwpzl3az5gr093uhvk7cwg348eajxsm9fjansur5qa97csfs597zh6`, this can be achieved running `sops --add-age ` + ## Install Nginx You need to install nginx (or apache2, or traefik, whichever you are more comfortable with, but be aware I'll only be able to help with nginx and traefik as I have no idea how apache2 works). Use ansible for this, and install the `nginx-extras` package. @@ -165,6 +209,8 @@ Create a new role and copy over these files to `roles/synapse/files`. Next customise the configuration file. If you feel like using postgres (you already set it up so why not, right ?) check the [docs](https://matrix-org.github.io/synapse/latest/usage/configuration/config_documentation.html#database). +Another approach that would be cleaner, but potentially harder, would be to tell ansible to run the configuration bootstrap command on the synapse machine if the `homeserver.yaml` file cannot be found, and then potentially customise the file. + ## Create the synapse docker container Normally your `synapse` module does not have any tasks yet, so go ahead and create a `roles/synapse/tasks/main.yml` file. You need to use the [`community.docker.docker_container`](https://docs.ansible.com/ansible/latest/collections/community/docker/docker_container_module.html#examples) module for this. In essence you need to make sure of a few things: @@ -181,11 +227,6 @@ For example, if I wanted to start a `redis` container that forwards 6379 and mou ```yaml --- -# This is needed because your host might not have it installed -- name: "Install docker module for python" - pip: - name: docker - executable: pip3 - name: creates the redis container community.docker.docker_container: name: "redis" @@ -276,7 +317,7 @@ Note about the matrix addresses, a matrix address looks like this: @: ``` -For example my address is @thomas:matrix.maurice.fr, the localpart is `thomas` and the homeserver is `matrix.maurice.fr` +For example my address is `@thomas:matrix.maurice.fr`, the localpart is `thomas` and the homeserver is `matrix.maurice.fr` In the following step you can create one or two users (for your team) to connect to the homeserver, pick the localpart carefully because that will be your username. @@ -303,3 +344,10 @@ Enter your hostname into the [matrix federation tester](https://federationtester #### Test by sending a message to another address Fire a message to `@thomas:matrix.maurice.fr`, or better yet, send a message to one of your colleague on another homeserver. + +### Congratulations you reached the end ! +This is pretty cool, however there is more you can do in the optional objectives of the lab, I would suggest you have a look at: +* Setup a Gitea runner and make sure your tests pass +* Add more tests ? +* Using a mechanism (sops/ansible vault/hashicorp vault) to store secrets in the repository and make it accessible to the gitea runner +* Finally, run ansible _from the CI_ so your playbooks get applied when you push new code to the repository \ No newline at end of file