mirror of
https://github.com/ppy/osu.git
synced 2024-11-16 09:17:48 +08:00
99bbbf810b
As is github tradition, workflows started yelling about running on a node version that was getting sunset, so here we go again. Relevant bumps: - https://github.com/actions/checkout/releases/tag/v4.0.0 - https://github.com/actions/setup-dotnet/releases/tag/v4.0.0 - https://github.com/actions/cache/releases/tag/v4.0.0 - https://github.com/actions/setup-java/releases/tag/v4.0.0 - https://github.com/peter-evans/create-pull-request/releases/tag/v6.0.0 - https://github.com/dorny/test-reporter/releases/tag/v1.8.0 Notably, `actions/upload-artifact` is _not_ bumped to v4, although it should be to resolve the node deprecation warnings, because it has more breaking changes and bumping would break `dorny/test-reporter` (see https://github.com/dorny/test-reporter/issues/363).
383 lines
15 KiB
YAML
383 lines
15 KiB
YAML
# ## Description
|
|
#
|
|
# Uses [diffcalc-sheet-generator](https://github.com/smoogipoo/diffcalc-sheet-generator) to run two builds of osu and generate an SR/PP/Score comparison spreadsheet.
|
|
#
|
|
# ## Requirements
|
|
#
|
|
# Self-hosted runner with installed:
|
|
# - `docker >= 20.10.16`
|
|
# - `docker-compose >= 2.5.1`
|
|
# - `lbzip2`
|
|
# - `jq`
|
|
#
|
|
# ## Usage
|
|
#
|
|
# The workflow can be run in two ways:
|
|
# 1. Via workflow dispatch.
|
|
# 2. By an owner of the repository posting a pull request or issue comment containing `!diffcalc`.
|
|
# For pull requests, the workflow will assume the pull request as the target to compare against (i.e. the `OSU_B` variable).
|
|
# Any lines in the comment of the form `KEY=VALUE` are treated as variables for the generator.
|
|
#
|
|
# ## Google Service Account
|
|
#
|
|
# Spreadsheets are uploaded to a Google Service Account, and exposed with read-only permissions to the wider audience.
|
|
#
|
|
# 1. Create a project at https://console.cloud.google.com
|
|
# 2. Enable the `Google Sheets` and `Google Drive` APIs.
|
|
# 3. Create a Service Account
|
|
# 4. Generate a key in the JSON format.
|
|
# 5. Encode the key as base64 and store as an **actions secret** with name **`DIFFCALC_GOOGLE_CREDENTIALS`**
|
|
#
|
|
# ## Environment variables
|
|
#
|
|
# The default environment may be configured via **actions variables**.
|
|
#
|
|
# Refer to [the sample environment](https://github.com/smoogipoo/diffcalc-sheet-generator/blob/master/.env.sample), and prefix each variable with `DIFFCALC_` (e.g. `DIFFCALC_THREADS`, `DIFFCALC_INNODB_BUFFER_SIZE`, etc...).
|
|
|
|
name: Run difficulty calculation comparison
|
|
|
|
run-name: "${{ github.event_name == 'workflow_dispatch' && format('Manual run: {0}', inputs.osu-b) || 'Automatic comment trigger' }}"
|
|
|
|
on:
|
|
issue_comment:
|
|
types: [ created ]
|
|
workflow_dispatch:
|
|
inputs:
|
|
osu-b:
|
|
description: "The target build of ppy/osu"
|
|
type: string
|
|
required: true
|
|
ruleset:
|
|
description: "The ruleset to process"
|
|
type: choice
|
|
required: true
|
|
options:
|
|
- osu
|
|
- taiko
|
|
- catch
|
|
- mania
|
|
converts:
|
|
description: "Include converted beatmaps"
|
|
type: boolean
|
|
required: false
|
|
default: true
|
|
ranked-only:
|
|
description: "Only ranked beatmaps"
|
|
type: boolean
|
|
required: false
|
|
default: true
|
|
generators:
|
|
description: "Comma-separated list of generators (available: [sr, pp, score])"
|
|
type: string
|
|
required: false
|
|
default: 'pp,sr'
|
|
osu-a:
|
|
description: "The source build of ppy/osu"
|
|
type: string
|
|
required: false
|
|
default: 'latest'
|
|
difficulty-calculator-a:
|
|
description: "The source build of ppy/osu-difficulty-calculator"
|
|
type: string
|
|
required: false
|
|
default: 'latest'
|
|
difficulty-calculator-b:
|
|
description: "The target build of ppy/osu-difficulty-calculator"
|
|
type: string
|
|
required: false
|
|
default: 'latest'
|
|
score-processor-a:
|
|
description: "The source build of ppy/osu-queue-score-statistics"
|
|
type: string
|
|
required: false
|
|
default: 'latest'
|
|
score-processor-b:
|
|
description: "The target build of ppy/osu-queue-score-statistics"
|
|
type: string
|
|
required: false
|
|
default: 'latest'
|
|
|
|
permissions:
|
|
pull-requests: write
|
|
|
|
env:
|
|
EXECUTION_ID: execution-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}
|
|
|
|
jobs:
|
|
check-permissions:
|
|
name: Check permissions
|
|
runs-on: ubuntu-latest
|
|
if: ${{ github.event_name == 'workflow_dispatch' || contains(github.event.comment.body, '!diffcalc') }}
|
|
steps:
|
|
- name: Check permissions
|
|
if: ${{ github.event_name != 'workflow_dispatch' }}
|
|
uses: actions-cool/check-user-permission@a0668c9aec87f3875fc56170b6452a453e9dd819 # v2.2.0
|
|
with:
|
|
require: 'write'
|
|
|
|
create-comment:
|
|
name: Create PR comment
|
|
needs: check-permissions
|
|
runs-on: ubuntu-latest
|
|
if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }}
|
|
steps:
|
|
- name: Create comment
|
|
uses: thollander/actions-comment-pull-request@363c6f6eae92cc5c3a66e95ba016fc771bb38943 # v2.4.2
|
|
with:
|
|
comment_tag: ${{ env.EXECUTION_ID }}
|
|
message: |
|
|
Difficulty calculation queued -- please wait! (${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
|
|
|
|
*This comment will update on completion*
|
|
|
|
directory:
|
|
name: Prepare directory
|
|
needs: check-permissions
|
|
runs-on: self-hosted
|
|
outputs:
|
|
GENERATOR_DIR: ${{ steps.set-outputs.outputs.GENERATOR_DIR }}
|
|
GENERATOR_ENV: ${{ steps.set-outputs.outputs.GENERATOR_ENV }}
|
|
GOOGLE_CREDS_FILE: ${{ steps.set-outputs.outputs.GOOGLE_CREDS_FILE }}
|
|
steps:
|
|
- name: Checkout diffcalc-sheet-generator
|
|
uses: actions/checkout@v4
|
|
with:
|
|
path: ${{ env.EXECUTION_ID }}
|
|
repository: 'smoogipoo/diffcalc-sheet-generator'
|
|
|
|
- name: Set outputs
|
|
id: set-outputs
|
|
run: |
|
|
echo "GENERATOR_DIR=${{ github.workspace }}/${{ env.EXECUTION_ID }}" >> "${GITHUB_OUTPUT}"
|
|
echo "GENERATOR_ENV=${{ github.workspace }}/${{ env.EXECUTION_ID }}/.env" >> "${GITHUB_OUTPUT}"
|
|
echo "GOOGLE_CREDS_FILE=${{ github.workspace }}/${{ env.EXECUTION_ID }}/google-credentials.json" >> "${GITHUB_OUTPUT}"
|
|
|
|
environment:
|
|
name: Setup environment
|
|
needs: directory
|
|
runs-on: self-hosted
|
|
env:
|
|
VARS_JSON: ${{ toJSON(vars) }}
|
|
steps:
|
|
- name: Add base environment
|
|
run: |
|
|
# Required by diffcalc-sheet-generator
|
|
cp '${{ needs.directory.outputs.GENERATOR_DIR }}/.env.sample' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
|
|
# Add Google credentials
|
|
echo '${{ secrets.DIFFCALC_GOOGLE_CREDENTIALS }}' | base64 -d > "${{ needs.directory.outputs.GOOGLE_CREDS_FILE }}"
|
|
|
|
# Add repository variables
|
|
echo "${VARS_JSON}" | jq -c '. | to_entries | .[]' | while read -r line; do
|
|
opt=$(jq -r '.key' <<< ${line})
|
|
val=$(jq -r '.value' <<< ${line})
|
|
|
|
if [[ "${opt}" =~ ^DIFFCALC_ ]]; then
|
|
optNoPrefix=$(echo "${opt}" | cut -d '_' -f2-)
|
|
sed -i "s;^${optNoPrefix}=.*$;${optNoPrefix}=${val};" "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
done
|
|
|
|
- name: Add pull-request environment
|
|
if: ${{ github.event_name == 'issue_comment' && github.event.issue.pull_request }}
|
|
run: |
|
|
sed -i "s;^OSU_B=.*$;OSU_B=${{ github.event.issue.pull_request.html_url }};" "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
|
|
- name: Add comment environment
|
|
if: ${{ github.event_name == 'issue_comment' }}
|
|
env:
|
|
COMMENT_BODY: ${{ github.event.comment.body }}
|
|
run: |
|
|
# Add comment environment
|
|
echo "$COMMENT_BODY" | sed -r 's/\r$//' | grep -E '^\w+=' | while read -r line; do
|
|
opt=$(echo "${line}" | cut -d '=' -f1)
|
|
sed -i "s;^${opt}=.*$;${line};" "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
done
|
|
|
|
- name: Add dispatch environment
|
|
if: ${{ github.event_name == 'workflow_dispatch' }}
|
|
run: |
|
|
sed -i 's;^OSU_B=.*$;OSU_B=${{ inputs.osu-b }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
sed -i 's/^RULESET=.*$/RULESET=${{ inputs.ruleset }}/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
sed -i 's/^GENERATORS=.*$/GENERATORS=${{ inputs.generators }}/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
|
|
if [[ '${{ inputs.osu-a }}' != 'latest' ]]; then
|
|
sed -i 's;^OSU_A=.*$;OSU_A=${{ inputs.osu-a }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.difficulty-calculator-a }}' != 'latest' ]]; then
|
|
sed -i 's;^DIFFICULTY_CALCULATOR_A=.*$;DIFFICULTY_CALCULATOR_A=${{ inputs.difficulty-calculator-a }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.difficulty-calculator-b }}' != 'latest' ]]; then
|
|
sed -i 's;^DIFFICULTY_CALCULATOR_B=.*$;DIFFICULTY_CALCULATOR_B=${{ inputs.difficulty-calculator-b }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.score-processor-a }}' != 'latest' ]]; then
|
|
sed -i 's;^SCORE_PROCESSOR_A=.*$;SCORE_PROCESSOR_A=${{ inputs.score-processor-a }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.score-processor-b }}' != 'latest' ]]; then
|
|
sed -i 's;^SCORE_PROCESSOR_B=.*$;SCORE_PROCESSOR_B=${{ inputs.score-processor-b }};' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.converts }}' == 'true' ]]; then
|
|
sed -i 's/^NO_CONVERTS=.*$/NO_CONVERTS=0/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
else
|
|
sed -i 's/^NO_CONVERTS=.*$/NO_CONVERTS=1/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
if [[ '${{ inputs.ranked-only }}' == 'true' ]]; then
|
|
sed -i 's/^RANKED_ONLY=.*$/RANKED_ONLY=1/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
else
|
|
sed -i 's/^RANKED_ONLY=.*$/RANKED_ONLY=0/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
fi
|
|
|
|
scores:
|
|
name: Setup scores
|
|
needs: [ directory, environment ]
|
|
runs-on: self-hosted
|
|
steps:
|
|
- name: Query latest data
|
|
id: query
|
|
run: |
|
|
ruleset=$(cat ${{ needs.directory.outputs.GENERATOR_ENV }} | grep -E '^RULESET=' | cut -d '=' -f2-)
|
|
performance_data_name=$(curl -s "https://data.ppy.sh/" | grep "performance_${ruleset}_top_1000\b" | tail -1 | awk -F "'" '{print $2}' | sed 's/\.tar\.bz2//g')
|
|
|
|
echo "TARGET_DIR=${{ needs.directory.outputs.GENERATOR_DIR }}/sql/${ruleset}" >> "${GITHUB_OUTPUT}"
|
|
echo "DATA_NAME=${performance_data_name}" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Restore cache
|
|
id: restore-cache
|
|
uses: maxnowack/local-cache@038cc090b52e4f205fbc468bf5b0756df6f68775 # v1
|
|
with:
|
|
path: ${{ steps.query.outputs.DATA_NAME }}.tar.bz2
|
|
key: ${{ steps.query.outputs.DATA_NAME }}
|
|
|
|
- name: Download
|
|
if: steps.restore-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
wget -q -nc "https://data.ppy.sh/${{ steps.query.outputs.DATA_NAME }}.tar.bz2"
|
|
|
|
- name: Extract
|
|
run: |
|
|
tar -I lbzip2 -xf "${{ steps.query.outputs.DATA_NAME }}.tar.bz2"
|
|
rm -r "${{ steps.query.outputs.TARGET_DIR }}"
|
|
mv "${{ steps.query.outputs.DATA_NAME }}" "${{ steps.query.outputs.TARGET_DIR }}"
|
|
|
|
beatmaps:
|
|
name: Setup beatmaps
|
|
needs: directory
|
|
runs-on: self-hosted
|
|
steps:
|
|
- name: Query latest data
|
|
id: query
|
|
run: |
|
|
beatmaps_data_name=$(curl -s "https://data.ppy.sh/" | grep "osu_files" | tail -1 | awk -F "'" '{print $2}' | sed 's/\.tar\.bz2//g')
|
|
|
|
echo "TARGET_DIR=${{ needs.directory.outputs.GENERATOR_DIR }}/beatmaps" >> "${GITHUB_OUTPUT}"
|
|
echo "DATA_NAME=${beatmaps_data_name}" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Restore cache
|
|
id: restore-cache
|
|
uses: maxnowack/local-cache@038cc090b52e4f205fbc468bf5b0756df6f68775 # v1
|
|
with:
|
|
path: ${{ steps.query.outputs.DATA_NAME }}.tar.bz2
|
|
key: ${{ steps.query.outputs.DATA_NAME }}
|
|
|
|
- name: Download
|
|
if: steps.restore-cache.outputs.cache-hit != 'true'
|
|
run: |
|
|
wget -q -nc "https://data.ppy.sh/${{ steps.query.outputs.DATA_NAME }}.tar.bz2"
|
|
|
|
- name: Extract
|
|
run: |
|
|
tar -I lbzip2 -xf "${{ steps.query.outputs.DATA_NAME }}.tar.bz2"
|
|
rm -r "${{ steps.query.outputs.TARGET_DIR }}"
|
|
mv "${{ steps.query.outputs.DATA_NAME }}" "${{ steps.query.outputs.TARGET_DIR }}"
|
|
|
|
generator:
|
|
name: Run generator
|
|
needs: [ directory, environment, scores, beatmaps ]
|
|
runs-on: self-hosted
|
|
timeout-minutes: 720
|
|
outputs:
|
|
TARGET: ${{ steps.run.outputs.TARGET }}
|
|
SPREADSHEET_LINK: ${{ steps.run.outputs.SPREADSHEET_LINK }}
|
|
steps:
|
|
- name: Run
|
|
id: run
|
|
run: |
|
|
# Add the GitHub token. This needs to be done here because it's unique per-job.
|
|
sed -i 's/^GH_TOKEN=.*$/GH_TOKEN=${{ github.token }}/' "${{ needs.directory.outputs.GENERATOR_ENV }}"
|
|
|
|
cd "${{ needs.directory.outputs.GENERATOR_DIR }}"
|
|
docker-compose up --build generator
|
|
|
|
link=$(docker-compose logs generator -n 10 | grep 'http' | sed -E 's/^.*(http.*)$/\1/')
|
|
target=$(cat "${{ needs.directory.outputs.GENERATOR_ENV }}" | grep -E '^OSU_B=' | cut -d '=' -f2-)
|
|
|
|
echo "TARGET=${target}" >> "${GITHUB_OUTPUT}"
|
|
echo "SPREADSHEET_LINK=${link}" >> "${GITHUB_OUTPUT}"
|
|
|
|
- name: Shutdown
|
|
if: ${{ always() }}
|
|
run: |
|
|
cd "${{ needs.directory.outputs.GENERATOR_DIR }}"
|
|
docker-compose down -v
|
|
|
|
output-cli:
|
|
name: Output info
|
|
needs: generator
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Output info
|
|
run: |
|
|
echo "Target: ${{ needs.generator.outputs.TARGET }}"
|
|
echo "Spreadsheet: ${{ needs.generator.outputs.SPREADSHEET_LINK }}"
|
|
|
|
cleanup:
|
|
name: Cleanup
|
|
needs: [ directory, generator ]
|
|
if: ${{ always() && needs.directory.result == 'success' }}
|
|
runs-on: self-hosted
|
|
steps:
|
|
- name: Cleanup
|
|
run: |
|
|
rm -rf "${{ needs.directory.outputs.GENERATOR_DIR }}"
|
|
|
|
update-comment:
|
|
name: Update PR comment
|
|
needs: [ create-comment, generator ]
|
|
runs-on: ubuntu-latest
|
|
if: ${{ always() && needs.create-comment.result == 'success' }}
|
|
steps:
|
|
- name: Update comment on success
|
|
if: ${{ needs.generator.result == 'success' }}
|
|
uses: thollander/actions-comment-pull-request@363c6f6eae92cc5c3a66e95ba016fc771bb38943 # v2.4.2
|
|
with:
|
|
comment_tag: ${{ env.EXECUTION_ID }}
|
|
mode: upsert
|
|
create_if_not_exists: false
|
|
message: |
|
|
Target: ${{ needs.generator.outputs.TARGET }}
|
|
Spreadsheet: ${{ needs.generator.outputs.SPREADSHEET_LINK }}
|
|
|
|
- name: Update comment on failure
|
|
if: ${{ needs.generator.result == 'failure' }}
|
|
uses: thollander/actions-comment-pull-request@363c6f6eae92cc5c3a66e95ba016fc771bb38943 # v2.4.2
|
|
with:
|
|
comment_tag: ${{ env.EXECUTION_ID }}
|
|
mode: upsert
|
|
create_if_not_exists: false
|
|
message: |
|
|
Difficulty calculation failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
|
|
- name: Update comment on cancellation
|
|
if: ${{ needs.generator.result == 'cancelled' }}
|
|
uses: thollander/actions-comment-pull-request@363c6f6eae92cc5c3a66e95ba016fc771bb38943 # v2.4.2
|
|
with:
|
|
comment_tag: ${{ env.EXECUTION_ID }}
|
|
mode: delete
|
|
message: '.' # Appears to be required by this action for non-error status code.
|