From ac1c4adf275124f10861de00a883febdd5fe9034 Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Tue, 28 Jan 2020 02:32:27 +0100 Subject: Remove outdated Gemfile.ci file to fix the tests Remove the Gemfile.ci file as it is outdated and not in use any longer. As the file is outdated, the tests also fail when using Gemfile.ci, thus removing the file will also fix the tests. Signed-off-by: Max Magorsch --- Gemfile.ci | 51 --------------------------------------------------- 1 file changed, 51 deletions(-) delete mode 100644 Gemfile.ci diff --git a/Gemfile.ci b/Gemfile.ci deleted file mode 100644 index eb5bb32..0000000 --- a/Gemfile.ci +++ /dev/null @@ -1,51 +0,0 @@ -source 'https://rubygems.org' - -# Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '5.2.0' -# Use mysql as the database for Active Record -# gem 'mysql2' -# Use SCSS for stylesheets -gem 'sass-rails', '~> 5.0' -# Use Uglifier as compressor for JavaScript assets -gem 'uglifier', '>= 1.3.0' - -# Use jquery as the JavaScript library -gem 'jquery-rails' -# Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks -gem 'turbolinks' -# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder -gem 'jbuilder', '~> 2.0' -# bundle exec rake doc:rails generates the API under doc/api. -gem 'sdoc', '~> 0.4.0', group: :doc - -# packages stuff -gem 'elasticsearch-rails' -gem 'elasticsearch-persistence' - -gem 'nokogiri' -gem 'parallel' -gem 'ruby-progressbar' -gem 'git' -gem 'thin' - -gem 'sinatra', require: false -gem 'sidekiq', require: false - -gem 'rdiscount' - -# Sidekiq bug? -gem 'did_you_mean', require: false # Sidekiq missing dep - -# UI -gem 'octicons_helper' - -group :development do - # Call 'byebug' anywhere in the code to stop execution and get a debugger console - gem 'byebug' - - # Access an IRB console on exception pages or by using <%= console %> in views - gem 'web-console', '~> 2.0' - - # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring - gem 'spring' -end -- cgit v1.2.3-65-gdbad From 17def0f9988e0b5bfb04c84861b597692b72d5c9 Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Wed, 5 Feb 2020 14:39:39 +0100 Subject: Migrate to gitlab-ci instead of travis Build and store the container images using gitlab. Signed-off-by: Max Magorsch --- .gitlab-ci.yml | 37 +++++++++++++++++++++++++++++++++++++ .travis.yml | 4 ++-- Dockerfile | 23 ++++++++++++++++++++++- 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..681d8e7 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,37 @@ +stages: + - build + - test + - dast + +info: + stage: .pre + script: + - docker info + +build: + stage: build + variables: + IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:$CI_COMMIT_SHA + LATEST_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_BRANCH:latest + script: + - echo $IMAGE_TAG + - echo $LATEST_IMAGE_TAG + - echo "$CI_REGISTRY_PASSWORD" | docker login -u "$CI_REGISTRY_USER" "$CI_REGISTRY" --password-stdin + - docker pull gentoo/portage:latest + - docker pull gentoo/stage3-amd64 + - docker pull $LATEST_IMAGE_TAG || true + - docker build --cache-from $LATEST_IMAGE_TAG -t $IMAGE_TAG -t $LATEST_IMAGE_TAG . + - docker push $LATEST_IMAGE_TAG + - docker push $IMAGE_TAG + +include: + - template: Dependency-Scanning.gitlab-ci.yml + - template: Container-Scanning.gitlab-ci.yml + - template: SAST.gitlab-ci.yml + - template: DAST.gitlab-ci.yml + +variables: + DS_DISABLE_DIND: "true" + SAST_DISABLE_DIND: "true" + DAST_WEBSITE: https://packagestest.gentoo.org +# DAST_FULL_SCAN_ENABLED: "true" diff --git a/.travis.yml b/.travis.yml index b8d7be3..b3b6d1e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ import: - source: .travis/travis.yml if: branch = master - - source: .travis/travis.docker.yml - if: branch = dev/main \ No newline at end of file +# - source: .travis/travis.docker.yml +# if: branch = dev/main diff --git a/Dockerfile b/Dockerfile index 55e6cb0..13d4f94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,24 @@ -FROM gentoo/rails:latest +FROM gentoo/portage:latest as portage +FROM gentoo/stage3-amd64 + +# Need a portage tree to build, use last nights. +COPY --from=portage /var/db/repos/gentoo /var/db/repos/gentoo + +# Sandbox doesn't work well in docker. +ENV FEATURES="-userpriv -usersandbox -sandbox" +ENV USE="-bindist" + +RUN emerge -C openssh +RUN emerge --quiet-build \ + net-libs/nodejs \ + dev-lang/ruby \ + dev-vcs/git +RUN ACCEPT_KEYWORDS="~amd64" emerge --quiet-build sys-apps/yarn + +RUN eselect ruby set ruby25 + +# Bundler is how we install the ruby stuff. +RUN gem install bundler -v 1.17.3 # Needed for changelogs. RUN git clone https://anongit.gentoo.org/git/repo/gentoo.git /mnt/packages-tree/gentoo/ @@ -17,3 +37,4 @@ RUN cp /var/www/packages.gentoo.org/htdocs/config/initializers/kkuleomi_config.r # Precompile our assets. RUN bundle exec rake webpacker:compile CMD ["bundler", "exec", "thin", "start"] + -- cgit v1.2.3-65-gdbad From 19a8b09d13b534b2742d236f65b7ee6a5301895c Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Sun, 9 Feb 2020 21:20:34 +0100 Subject: Prepare docker-compose.yml for usage in production Signed-off-by: Max Magorsch --- docker-compose.yml | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c662d13..cba9792 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,11 +17,9 @@ version: '3.2' # services: http-serving: - # Build from Dockerfile in . - image: gentoo/gpackages:latest - build: . + image: ${GPACKAGES_IMAGE:-gentoo/packages:latest} ports: - - 5000 + - 127.0.0.1:5000:5000 volumes: - type: volume source: portage @@ -38,14 +36,14 @@ services: - RAILS_ENV=production - NODE_ENV=production - MEMCACHE_URL="memcache:11211" - - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50 + - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50} + restart: always depends_on: - redis - elasticsearch command: bash -c "cp config/initializers/kkuleomi_config.rb.dist config/initializers/kkuleomi_config.rb && /var/www/packages.gentoo.org/htdocs/bin/first-run production" sidekiq: - image: gentoo/gpackages:latest - build: . + image: ${GPACKAGES_IMAGE:-gentoo/packages:latest} volumes: - type: volume source: portage @@ -60,7 +58,8 @@ services: - REDIS_URL=redis://redis:6379 - MEMCACHE_URL="memcache:11211" - ELASTICSEARCH_URL=elasticsearch:9200 - - SECRET_KEY_BASE=6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50 + - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50} + restart: always depends_on: - redis - elasticsearch @@ -68,21 +67,29 @@ services: bash -c " cp config/initializers/kkuleomi_config.rb.dist config/initializers/kkuleomi_config.rb && bundler install && yarn install --check-files && bundle exec sidekiq -c 5" memcache: - image: memcached:latest - ports: - - 11211 + image: memcached:${MEMCACHED_VERSION:-1.5} + restart: always elasticsearch: # TODO(antarus): We should build a docker image for this based on gentoo. - image: docker.elastic.co/elasticsearch/elasticsearch:7.3.1 - # Run in single-node config. + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-7.3.1} + volumes: + - ${ELASTICSEARCH_DATA_PATH:-/var/lib/elasticsearch/data/}:/usr/share/elasticsearch/data/ environment: - - discovery.type=single-node - ports: - - 9200 + - node.name=packages + - cluster.initial_master_nodes=packages + - cluster.routing.allocation.node_initial_primaries_recoveries=8 + - cluster.routing.allocation.node_concurrent_outgoing_recoveries=8 + - cluster.routing.allocation.node_concurrent_incoming_recoveries=8 + - indices.recovery.max_bytes_per_sec=200mb + - "ES_JAVA_OPTS=-Xms4g -Xmx4g" + ulimits: + memlock: + soft: -1 + hard: -1 + restart: always redis: - image: redis:4.0.6 - ports: - - 6379 + image: redis:${REDIS_VERSION:-4.0.6} + restart: always volumes: portage: -- cgit v1.2.3-65-gdbad From 1ab83e61da620442c5d5c6f44e052d9526593eee Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Mon, 10 Feb 2020 04:38:04 +0100 Subject: Add watchtower to automatically deploy new images Watchtower will automatically check whether a new version of a container image is present in the specified container registry. If so, watchtower will automatically pull the new image and restart the container. By default, watchtower will look for new images every 5 minutes. Signed-off-by: Max Magorsch --- docker-compose.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index cba9792..73a3538 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,6 +38,8 @@ services: - MEMCACHE_URL="memcache:11211" - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50} restart: always + labels: + com.centurylinklabs.watchtower.enable: "true" depends_on: - redis - elasticsearch @@ -60,6 +62,8 @@ services: - ELASTICSEARCH_URL=elasticsearch:9200 - SECRET_KEY_BASE= ${SECRET_KEY_BASE:-6c9710aeb74dd88ff1d1b8f4bd6d7d8e0f340905d0974400fffd7246714aa703cf7bf4a98c0bc90317a3b803b82c0f9371e18ada19fc4eed9d6118077a249f50} restart: always + labels: + com.centurylinklabs.watchtower.enable: "true" depends_on: - redis - elasticsearch @@ -90,6 +94,12 @@ services: redis: image: redis:${REDIS_VERSION:-4.0.6} restart: always + watchtower: + image: containrrr/watchtower:0.3.10 + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /root/.docker/config.json:/config.json + command: --label-enable volumes: portage: -- cgit v1.2.3-65-gdbad From 71229c2c5cad839fc6a0c89cc6598e78bbd77b0a Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Mon, 10 Feb 2020 05:32:46 +0100 Subject: Minor fix in the README Signed-off-by: Max Magorsch --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 26ec7f4..c42d319 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ internally codenamed kkuleomi/꾸러미 which is Korean for package (who would h ## Build status -* [![Travis-CI Build Status](https://travis-ci.org/gentoo/gpackages.svg?branch=master)](https://travis-ci.org/gentoo/gpackages) +[![Travis-CI Build Status](https://travis-ci.org/gentoo/gpackages.svg?branch=master)](https://travis-ci.org/gentoo/gpackages) ## Installation instructions -- cgit v1.2.3-65-gdbad From 087a47c4842ddd9b13cb81762fc6161bcf20d4b4 Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Tue, 11 Feb 2020 22:32:31 +0100 Subject: Explicitly call /changelog.html instead of /changelog This is fixing problems that have been arised in production when using the app in combination with nginx. Signed-off-by: Max Magorsch --- app/webpack/src/javascript/packages/show.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/webpack/src/javascript/packages/show.js b/app/webpack/src/javascript/packages/show.js index c964fdb..d3c63a4 100644 --- a/app/webpack/src/javascript/packages/show.js +++ b/app/webpack/src/javascript/packages/show.js @@ -2,7 +2,7 @@ $(function() { var atom = $('#package-title').data('atom'); $.ajax({ - url: '/packages/' + atom + '/changelog' + url: '/packages/' + atom + '/changelog.html' }).done(function(data) { $('#changelog-container').html(data); $(document).trigger('kkuleomi:ajax'); -- cgit v1.2.3-65-gdbad From e15e53592db6fe2535500b0c4a952ec374652207 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Tue, 11 Feb 2020 17:42:31 -0800 Subject: view: switch to relative favicon Signed-off-by: Robin H. Johnson --- app/views/layouts/application.html.erb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index a22f4f5..0565223 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -8,7 +8,7 @@ Gentoo Packages Database"> <%= stylesheet_pack_tag 'application' %> <%= javascript_pack_tag 'application' %> - + <% if content_for? :head -%> <%= yield :head %> <% end -%> -- cgit v1.2.3-65-gdbad From cc54af93b71fe06d91b9afb625fc225b0686f2b7 Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Wed, 12 Feb 2020 03:14:15 +0100 Subject: Add the current version to the footer Signed-off-by: Max Magorsch --- app/views/layouts/application.html.erb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 0565223..4562675 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -108,6 +108,7 @@

Questions or comments?

Please feel free to contact us. +

<%= `git describe --tags` %>

-- cgit v1.2.3-65-gdbad From 2f5b6df2a88571a4f55f85a14f5cba258bb70724 Mon Sep 17 00:00:00 2001 From: "Robin H. Johnson" Date: Wed, 12 Feb 2020 11:50:49 -0800 Subject: views: rip out Piwik Signed-off-by: Robin H. Johnson --- app/views/layouts/application.html.erb | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/app/views/layouts/application.html.erb b/app/views/layouts/application.html.erb index 4562675..ce9dda9 100644 --- a/app/views/layouts/application.html.erb +++ b/app/views/layouts/application.html.erb @@ -131,19 +131,5 @@
- - -- cgit v1.2.3-65-gdbad From 6643292badd88e68256f3a6b9f23cc65f82b417e Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Wed, 12 Feb 2020 23:21:30 +0100 Subject: Don't recreate the ES-index by default By default the ES index will only be created now if it is not alredy present. In case it is already present it won't be recreated when using the kkuleomi:index:init task. To forcefully recreate the index, kkuleomi:index:recreate can be used. Signed-off-by: Max Magorsch --- lib/kkuleomi/store.rb | 2 +- lib/tasks/kkuleomi.rake | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb index 9640a87..ff8cb40 100644 --- a/lib/kkuleomi/store.rb +++ b/lib/kkuleomi/store.rb @@ -13,7 +13,7 @@ module Kkuleomi::Store # In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own. repositories.each { |repository| - repository.instance.create_index!(force: true, settings: settings) + repository.instance.create_index!(force: _force, settings: settings) } end end diff --git a/lib/tasks/kkuleomi.rake b/lib/tasks/kkuleomi.rake index 9362b7a..45d9283 100644 --- a/lib/tasks/kkuleomi.rake +++ b/lib/tasks/kkuleomi.rake @@ -2,6 +2,10 @@ namespace :kkuleomi do namespace :index do desc '(Re-)Initializes the ElasticSearch index' task init: :environment do + Kkuleomi::Store.create_index false + end + + task recreate: :environment do Kkuleomi::Store.create_index true end end -- cgit v1.2.3-65-gdbad From 393157da89ce3df31f4019e5d70e52fcf6bd9cce Mon Sep 17 00:00:00 2001 From: Max Magorsch Date: Wed, 12 Feb 2020 23:26:31 +0100 Subject: Don't install a crontab in the containers Signed-off-by: Max Magorsch --- bin/first-run | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bin/first-run b/bin/first-run index f850cc8..5fbe671 100755 --- a/bin/first-run +++ b/bin/first-run @@ -11,10 +11,5 @@ bundle exec rake webpacker:compile RAILS_ENV=production bundle exec rake kkuleomi:index:init RAILS_ENV=${1:-development} ./bin/update-all.sh ${1:-development} -if [[ "${1:-development}" == "production" ]] -then - crontab -l | { cat; echo "*/10 * * * * /var/www/packages.gentoo.org/htdocs/bin/update-all.sh ${1:-development}"; } | crontab - -fi - # Finally start the http server when the index is initialized bundle exec thin start -p 5000 \ No newline at end of file -- cgit v1.2.3-65-gdbad