diff options
44 files changed, 819 insertions, 505 deletions
@@ -23,8 +23,8 @@ gem 'jbuilder', '~> 2.0' gem 'sdoc', '~> 1.0', group: :doc # packages stuff -gem 'elasticsearch-rails', '~> 5.0' -gem 'elasticsearch-persistence', '~> 5.0' +gem 'elasticsearch-rails', '~> 7.0.0' +gem 'elasticsearch-persistence', '~> 7.0.0' gem 'nokogiri' gem 'thin' diff --git a/Gemfile.lock b/Gemfile.lock index 78774c0..14ab1e1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -44,42 +44,32 @@ GEM tzinfo (~> 1.1) arel (9.0.0) ast (2.4.0) - axiom-types (0.1.1) - descendants_tracker (~> 0.0.4) - ice_nine (~> 0.11.0) - thread_safe (~> 0.3, >= 0.3.1) bindex (0.8.1) builder (3.2.3) byebug (11.0.1) - coercible (1.0.0) - descendants_tracker (~> 0.0.1) concurrent-ruby (1.1.5) connection_pool (2.2.2) crass (1.0.4) daemons (1.3.1) - descendants_tracker (0.0.4) - thread_safe (~> 0.3, >= 0.3.1) - elasticsearch (5.0.5) - elasticsearch-api (= 5.0.5) - elasticsearch-transport (= 5.0.5) - elasticsearch-api (5.0.5) + elasticsearch (7.3.0) + elasticsearch-api (= 7.3.0) + elasticsearch-transport (= 7.3.0) + elasticsearch-api (7.3.0) multi_json - elasticsearch-model (5.1.0) + elasticsearch-model (7.0.0) activesupport (> 3) - elasticsearch (~> 5) + elasticsearch (> 1) hashie - elasticsearch-persistence (5.1.0) + elasticsearch-persistence (7.0.0) activemodel (> 4) activesupport (> 4) - elasticsearch (~> 5) - elasticsearch-model (~> 5) + elasticsearch (~> 7) + elasticsearch-model (= 7.0.0) hashie - virtus - elasticsearch-rails (5.1.0) - elasticsearch-transport (5.0.5) + elasticsearch-rails (7.0.0) + elasticsearch-transport (7.3.0) faraday multi_json - equalizer (0.0.11) erubi (1.8.0) eventmachine (1.2.7) execjs (2.7.0) @@ -91,7 +81,6 @@ GEM hashie (3.6.0) i18n (1.6.0) concurrent-ruby (~> 1.0) - ice_nine (0.11.2) jaro_winkler (1.5.3) jbuilder (2.9.1) activesupport (>= 4.2.0) @@ -222,11 +211,6 @@ GEM uglifier (4.1.20) execjs (>= 0.3.0, < 3) unicode-display_width (1.6.0) - virtus (1.0.5) - axiom-types (~> 0.1) - coercible (~> 1.0) - descendants_tracker (~> 0.0, >= 0.0.3) - equalizer (~> 0.0, >= 0.0.9) web-console (3.7.0) actionview (>= 5.0) activemodel (>= 5.0) @@ -241,8 +225,8 @@ PLATFORMS DEPENDENCIES byebug - elasticsearch-persistence (~> 5.0) - elasticsearch-rails (~> 5.0) + elasticsearch-persistence (~> 7.0.0) + elasticsearch-rails (~> 7.0.0) jbuilder (~> 2.0) jquery-rails (~> 4.3.5) listen diff --git a/app/controllers/arches_controller.rb b/app/controllers/arches_controller.rb index cbbcb65..c72e378 100644 --- a/app/controllers/arches_controller.rb +++ b/app/controllers/arches_controller.rb @@ -43,9 +43,9 @@ class ArchesController < ApplicationController def keyworded_packages(arch) Rails.cache.fetch("keyworded_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'keyword', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'keyword', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -53,9 +53,9 @@ class ArchesController < ApplicationController def stabled_packages(arch) Rails.cache.fetch("stabled_packages/#{arch}", expires_in: 10.minutes) do - Change.filter_all({ change_type: 'stable', arches: arch }, - size: 50, - sort: { created_at: { order: 'desc' } }).map do |change| + ChangeRepository.filter_all({ change_type: 'stable', arches: arch }, + size: 50, + sort: { created_at: { order: 'desc' } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/categories_controller.rb b/app/controllers/categories_controller.rb index 33817aa..a9c9b06 100644 --- a/app/controllers/categories_controller.rb +++ b/app/controllers/categories_controller.rb @@ -3,15 +3,15 @@ class CategoriesController < ApplicationController before_action :set_nav def index - @categories = Category.all_sorted_by(:name, :asc) + @categories = CategoryRepository.all_sorted_by(:id, :asc) end def show @packages = Rails.cache.fetch("category/#{@category.name}/packages", expires_in: 10.minutes) do - Package.find_all_by(:category, - @category.name, - sort: { name_sort: { order: 'asc' } }).map do |pkg| + PackageRepository.find_all_by(:category, + @category.name, + sort: { name_sort: { order: 'asc' } }).map do |pkg| pkg.to_os(:name, :atom, :description) end end @@ -24,7 +24,7 @@ class CategoriesController < ApplicationController private def set_category - @category = Category.find_by(:name, params[:id]) + @category = CategoryRepository.find_by(:name, params[:id]) fail ActionController::RoutingError, 'No such category' unless @category @title = @category.name diff --git a/app/controllers/concerns/package_update_feeds.rb b/app/controllers/concerns/package_update_feeds.rb index 2d20672..28a951b 100644 --- a/app/controllers/concerns/package_update_feeds.rb +++ b/app/controllers/concerns/package_update_feeds.rb @@ -3,7 +3,7 @@ module PackageUpdateFeeds def new_packages Rails.cache.fetch('new_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'new_package', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :created_at) end end @@ -11,7 +11,7 @@ module PackageUpdateFeeds def version_bumps Rails.cache.fetch('version_bumps', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'version_bump', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :created_at) end end @@ -19,7 +19,7 @@ module PackageUpdateFeeds def keyworded_packages Rails.cache.fetch('keyworded_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'keyword', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end @@ -27,7 +27,7 @@ module PackageUpdateFeeds def stabled_packages Rails.cache.fetch('stabled_packages', expires_in: 10.minutes) do - Change.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| + ChangeRepository.find_all_by(:change_type, 'stable', { size: 50, sort: { created_at: { order: 'desc' } } }).map do |change| change.to_os(:change_type, :package, :category, :version, :arches, :created_at) end end diff --git a/app/controllers/packages_controller.rb b/app/controllers/packages_controller.rb index 64cb289..67cc86f 100644 --- a/app/controllers/packages_controller.rb +++ b/app/controllers/packages_controller.rb @@ -8,24 +8,24 @@ class PackagesController < ApplicationController def search @offset = params[:o].to_i || 0 - @packages = Package.default_search(params[:q], @offset) + @packages = PackageRepository.default_search(params[:q], @offset) redirect_to package_path(@packages.first).gsub('%2F', '/') if @packages.size == 1 end def suggest - @packages = Package.suggest(params[:q]) + @packages = PackageRepository.suggest(params[:q]) end def resolve - @packages = Package.resolve(params[:atom]) + @packages = PackageRepository.resolve(params[:atom]) end def show - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - fresh_when etag: @package.updated_at, last_modified: @package.updated_at, public: true + fresh_when etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true # Enable this in 2024 (when we have full-color emojis on a Linux desktop) # @title = ' 📦 %s' % @package.atom @@ -34,10 +34,10 @@ class PackagesController < ApplicationController end def changelog - @package = Package.find_by(:atom, params[:id]) + @package = PackageRepository.find_by(:atom, params[:id]) fail ActionController::RoutingError, 'No such package' unless @package - if stale?(etag: @package.updated_at, last_modified: @package.updated_at, public: true) + if stale?(etag: Time.parse(@package.updated_at), last_modified: Time.parse(@package.updated_at), public: true) @changelog = Rails.cache.fetch("changelog/#{@package.atom}") do Portage::Util::History.for(@package.category, @package.name, 5) end diff --git a/app/controllers/useflags_controller.rb b/app/controllers/useflags_controller.rb index 0fa74f4..9802b78 100644 --- a/app/controllers/useflags_controller.rb +++ b/app/controllers/useflags_controller.rb @@ -6,18 +6,18 @@ class UseflagsController < ApplicationController end def show - @useflags = Useflag.get_flags(params[:id]) + @useflags = UseflagRepository.get_flags(params[:id]) if @useflags.empty? || (@useflags[:use_expand].empty? && @useflags[:local].empty? && @useflags[:global].empty?) fail ActionController::RoutingError, 'No such useflag' end - @packages = Package.find_atoms_by_useflag(params[:id]) + @packages = PackageRepository.find_atoms_by_useflag(params[:id]) @title = '%s – %s' % [params[:id], t(:use_flags)] unless @useflags[:use_expand].empty? @useflag = @useflags[:use_expand].first - @use_expand_flags = Useflag.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) + @use_expand_flags = UseflagRepository.find_all_by(:use_expand_prefix, @useflag.use_expand_prefix) @use_expand_flag_name = @useflag.use_expand_prefix.upcase render template: 'useflags/show_use_expand' @@ -29,16 +29,16 @@ class UseflagsController < ApplicationController def search # TODO: Different search? - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def suggest - @flags = Useflag.suggest(params[:q]) + @flags = UseflagRepository.suggest(params[:q]) end def popular @popular_useflags = Rails.cache.fetch('popular_useflags', expires_in: 24.hours) do - Version.get_popular_useflags(100) + VersionRepository.get_popular_useflags(100) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index 619582c..8405e59 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -37,6 +37,9 @@ module ApplicationHelper end def i18n_date(date, format = '%a, %e %b %Y %H:%M') + + date = Time.parse(date).utc if date.is_a? String + content_tag :span, l(date, format: format), class: 'kk-i18n-date', diff --git a/app/jobs/category_update_job.rb b/app/jobs/category_update_job.rb index 7443099..e764ad8 100644 --- a/app/jobs/category_update_job.rb +++ b/app/jobs/category_update_job.rb @@ -5,8 +5,8 @@ class CategoryUpdateJob < ApplicationJob category_path, options = args model = Portage::Repository::Category.new(category_path) - category = Category.find_by(:name, model.name) || Category.new - idx_packages = Package.find_all_by(:category, model.name) || [] + category = CategoryRepository.find_by(:name, model.name) || Category.new + idx_packages = PackageRepository.find_all_by(:category, model.name) || [] if category.needs_import? model category.import! model diff --git a/app/jobs/package_removal_job.rb b/app/jobs/package_removal_job.rb index 877ed07..e625b96 100644 --- a/app/jobs/package_removal_job.rb +++ b/app/jobs/package_removal_job.rb @@ -4,11 +4,11 @@ class PackageRemovalJob < ApplicationJob def perform(*args) atom, _options = args - package_doc = Package.find_by(:atom, atom) + package_doc = PackageRepository.find_by(:atom, atom) return if package_doc.nil? - package_doc.versions.each(&:delete) - package_doc.delete + package_doc.versions.each { |v| VersionRepository.delete(v) } + PackageRepository.delete(package_doc) Rails.logger.warn { "Package deleted: #{atom}" } # USE flags are cleaned up by the UseflagsUpdateJob diff --git a/app/jobs/package_update_job.rb b/app/jobs/package_update_job.rb index 55e278f..53a352c 100644 --- a/app/jobs/package_update_job.rb +++ b/app/jobs/package_update_job.rb @@ -4,7 +4,7 @@ class PackageUpdateJob < ApplicationJob def perform(*args) path, options = args package_model = Portage::Repository::Package.new(path) - package_doc = Package.find_by(:atom, package_model.to_cp) || Package.new + package_doc = PackageRepository.find_by(:atom, package_model.to_cp) || Package.new if package_doc.needs_import? package_model package_doc.import!(package_model, options) diff --git a/app/jobs/record_change_job.rb b/app/jobs/record_change_job.rb index 0e6a011..ed5dd5e 100644 --- a/app/jobs/record_change_job.rb +++ b/app/jobs/record_change_job.rb @@ -25,6 +25,6 @@ class RecordChangeJob < ApplicationJob c.change_type = 'removal' end - c.save + ChangeRepository.save(c) end end diff --git a/app/jobs/useflags_update_job.rb b/app/jobs/useflags_update_job.rb index 21145c3..5558d47 100644 --- a/app/jobs/useflags_update_job.rb +++ b/app/jobs/useflags_update_job.rb @@ -10,7 +10,7 @@ class UseflagsUpdateJob < ApplicationJob def update_global(repo) model_flags = repo.global_useflags - index_flags = Useflag.global + index_flags = UseflagRepository.global new_flags = model_flags.keys - index_flags.keys del_flags = index_flags.keys - model_flags.keys @@ -21,24 +21,24 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.name = flag flag_doc.description = model_flags[flag] flag_doc.scope = 'global' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def update_use_expand(repo) model_flags = repo.use_expand_flags - index_flags = Useflag.use_expand + index_flags = UseflagRepository.use_expand # Calculate keys only once index_flag_keys = index_flags.keys @@ -55,7 +55,7 @@ class UseflagsUpdateJob < ApplicationJob if index_flag_keys.include? _flag unless index_flags[_flag].description == desc index_flags[_flag].description = desc - index_flags[_flag].save + UseflagRepository.save(index_flags[_flag]) end else # New flag @@ -64,14 +64,14 @@ class UseflagsUpdateJob < ApplicationJob flag_doc.description = desc flag_doc.scope = 'use_expand' flag_doc.use_expand_prefix = variable - flag_doc.save + UseflagRepository.save(flag_doc) end end end # Find and process removed flags flag_status.each_pair do |flag, status| - index_flags[flag].delete unless status + UseflagRepository.delete(index_flags[flag]) unless status end end diff --git a/app/models/category.rb b/app/models/category.rb index f629bde..4e361c1 100644 --- a/app/models/category.rb +++ b/app/models/category.rb @@ -1,12 +1,39 @@ class Category - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "categories-#{Rails.env}" + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @name + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :metadata_hash, String, mapping: { type: 'text' } # Determines if the document model needs an update from the repository model # @@ -29,7 +56,7 @@ class Category # @param [Portage::Repository::Category] category_model Input category model def import!(category_model) import(category_model) - save + CategoryRepository.save(self) end # Returns the URL parameter for referencing this package (Rails internal stuff) diff --git a/app/models/change.rb b/app/models/change.rb index 6eaf00c..1793da4 100644 --- a/app/models/change.rb +++ b/app/models/change.rb @@ -1,13 +1,48 @@ class Change - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations - index_name "change-#{Rails.env}" + ATTRIBUTES = [:_id, + :created_at, + :updated_at, + :package, + :category, + :change_type, + :version, + :arches, + :commit] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :package, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end - attribute :package, String, mapping: { type: 'keyword' } - attribute :category, String, mapping: { type: 'keyword' } - attribute :change_type, String, mapping: { type: 'keyword' } - attribute :version, String, mapping: { type: 'keyword' } - attribute :arches, String, mapping: { type: 'keyword' } - attribute :commit, Hash, default: {}, mapping: { type: 'object' } end diff --git a/app/models/package.rb b/app/models/package.rb index 7ad3cbe..11ef135 100644 --- a/app/models/package.rb +++ b/app/models/package.rb @@ -1,31 +1,52 @@ class Package - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::PackageImport - include Kkuleomi::Store::Models::PackageSearch - - index_name "packages-#{Rails.env}" - - raw_fields = { - type: 'keyword' - } - - attribute :category, String, mapping: raw_fields - attribute :name, String, mapping: raw_fields - attribute :name_sort, String, mapping: raw_fields - attribute :atom, String, mapping: raw_fields - attribute :description, String, mapping: { type: 'text' } - attribute :longdescription, String, mapping: { type: 'text' } - attribute :homepage, String, default: [], mapping: raw_fields - attribute :license, String, mapping: raw_fields - attribute :licenses, String, default: [], mapping: raw_fields - attribute :herds, String, default: [], mapping: raw_fields - attribute :maintainers, Array, default: [], mapping: { type: 'object' } - attribute :useflags, Hash, default: {}, mapping: { type: 'object' } - attribute :metadata_hash, String, mapping: raw_fields + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :category, + :name, + :name_sort, + :atom, + :description, + :longdescription, + :homepage, + :license, + :licenses, + :herds, + :maintainers, + :useflags, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def category_model - @category_model ||= Category.find_by(:name, category) + @category_model ||= CategoryRepository.find_by(:name, category) end def to_param @@ -44,7 +65,7 @@ class Package end def versions - @versions ||= Version.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) + @versions ||= VersionRepository.find_all_by(:package, atom, sort: { sort_key: { order: 'asc' } }) end def latest_version @@ -65,6 +86,15 @@ class Package maintainers.empty? && herds.empty? end + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + private # Splits a license string into single licenses, stripping the permitted logic constructs diff --git a/app/models/useflag.rb b/app/models/useflag.rb index 131a89c..12758cb 100644 --- a/app/models/useflag.rb +++ b/app/models/useflag.rb @@ -1,14 +1,41 @@ class Useflag - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model - - index_name "useflags-#{Rails.env}" + include ActiveModel::Model + include ActiveModel::Validations + + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :name, + :description, + :atom, + :scope, + :use_expand_prefix] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :name, presence: true + + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end - attribute :name, String, mapping: { type: 'keyword' } - attribute :description, String, mapping: { type: 'text' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :scope, String, mapping: { type: 'keyword' } - attribute :use_expand_prefix, String, mapping: { type: 'keyword' } + def attributes + @id = @name + '-' + (@atom || 'global' ) + '-' + @scope + @created_at ||= DateTime.now + @updated_at = DateTime.now + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes def all_fields [:name, :description, :atom, :scope, :use_expand_prefix] @@ -22,78 +49,14 @@ class Useflag name.gsub(use_expand_prefix + '_', '') end - class << self - # Retrieves all flags sorted by their state - def get_flags(name) - result = { local: {}, global: [], use_expand: [] } - - find_all_by(:name, name).each do |flag| - case flag.scope - when 'local' - result[:local][flag.atom] = flag - when 'global' - result[:global] << flag - when 'use_expand' - result[:use_expand] << flag - end - end - - result - end - - def suggest(q) - results = Useflag.search( - size: 20, - query: { match_phrase_prefix: { name: q } } - ) - - processed_results = {} - results.each do |result| - if processed_results.key? result.name - processed_results[result.name] = { - name: result.name, - description: '(multiple definitions)', - scope: 'multi' - } - else - processed_results[result.name] = result - end - end - - processed_results.values.sort { |a, b| a[:name].length <=> b[:name].length } - end - - # Loads the local USE flags for a given package in a name -> model hash - # - # @param [String] atom Package to find flags for - # @return [Hash] - def local_for(atom) - map_by_name find_all_by(:atom, atom) - end - - # Maps the global USE flags in the index by their name - # This is expensive! - # - def global - map_by_name find_all_by(:scope, 'global') - end - - # Maps the USE_EXPAND variables in the index by their name - # - def use_expand - map_by_name find_all_by(:scope, 'use_expand') - end - - private + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end - def map_by_name(collection) - map = {} - collection.each do |item| - map[item.name] = item - end - - map - end - end end diff --git a/app/models/version.rb b/app/models/version.rb index 62c72f8..3629b98 100644 --- a/app/models/version.rb +++ b/app/models/version.rb @@ -1,23 +1,52 @@ +require 'date' + class Version - include Elasticsearch::Persistence::Model - include Kkuleomi::Store::Model + include ActiveModel::Model + include ActiveModel::Validations include Kkuleomi::Store::Models::VersionImport - index_name "versions-#{Rails.env}" - - attribute :version, String, mapping: { type: 'keyword' } - attribute :package, String, mapping: { type: 'keyword' } - attribute :atom, String, mapping: { type: 'keyword' } - attribute :sort_key, Integer, mapping: { type: 'integer' } - attribute :slot, String, mapping: { type: 'keyword' } - attribute :subslot, String, mapping: { type: 'keyword' } - attribute :eapi, String, mapping: { type: 'keyword' } - attribute :keywords, String, mapping: { type: 'keyword' } - attribute :masks, Array, default: [], mapping: { type: 'object' } - attribute :use, String, default: [], mapping: { type: 'keyword' } - attribute :restrict, String, default: [], mapping: { type: 'keyword' } - attribute :properties, String, default: [], mapping: { type: 'keyword' } - attribute :metadata_hash, String, mapping: { type: 'keyword' } + ATTRIBUTES = [:id, + :created_at, + :updated_at, + :version, + :package, + :atom, + :sort_key, + :slot, + :subslot, + :eapi, + :keywords, + :masks, + :use, + :restrict, + :properties, + :metadata_hash] + attr_accessor(*ATTRIBUTES) + attr_reader :attributes + + validates :version, presence: true + + def initialize(attr={}) + attr.each do |k,v| + if ATTRIBUTES.include?(k.to_sym) + send("#{k}=", v) + end + end + end + + def attributes + @id = @atom + @created_at ||= DateTime.now + @updated_at = DateTime.now + + ATTRIBUTES.inject({}) do |hash, attr| + if value = send(attr) + hash[attr] = value + end + hash + end + end + alias :to_hash :attributes # Returns the keywording state on a given architecture # @@ -138,14 +167,14 @@ class Version def calc_useflags result = { local: {}, global: {}, use_expand: {} } - local_flag_map = Useflag.local_for(atom.gsub("-#{version}", '')) + local_flag_map = UseflagRepository.local_for(atom.gsub("-#{version}", '')) local_flags = local_flag_map.keys use.sort.each do |flag| if local_flags.include? flag result[:local][flag] = local_flag_map[flag].to_hsh else - useflag = Useflag.find_by(:name, flag) + useflag = UseflagRepository.find_by(:name, flag) # This should not happen, but let's be sure next unless useflag diff --git a/app/repositories/base_repository.rb b/app/repositories/base_repository.rb new file mode 100644 index 0000000..7154691 --- /dev/null +++ b/app/repositories/base_repository.rb @@ -0,0 +1,88 @@ +require 'forwardable' +require 'singleton' + +class BaseRepository + include Elasticsearch::Persistence::Repository + include Elasticsearch::Persistence::Repository::DSL + include Singleton + + client ElasticsearchClient.default + + class << self + extend Forwardable + def_delegators :instance, :find_all_by, :filter_all, :find_by, :find_all_by_parent, :all_sorted_by + def_delegators :instance, :count, :search, :delete, :save, :refresh_index!, :create_index + end + + # Finds instances by exact IDs using the 'term' filter + def find_all_by(field, value, opts = {}) + search({ + size: 10_000, + query: { match: { field => value } } + }.merge(opts)) + end + + # Filter all instances by the given parameters + def filter_all(filters, opts = {}) + filter_args = [] + filters.each_pair { |field, value| filter_args << { term: { field => value } } } + + search({ + query: { + bool: { filter: { bool: { must: filter_args } } } + }, + size: 10_000 + }.merge(opts)) + end + + def find_by(field, value, opts = {}) + find_all_by(field, value, opts).first + end + + def find_all_by_parent(parent, opts = {}) + search(opts.merge( + size: 10_000, + query: { + bool: { + filter: { + has_parent: { + parent_type: parent.class.document_type, + query: { term: { _id: parent.id } } + } + }, + must: { + match_all: {} + } + } + }) + ) + end + + # Returns all (by default 10k) records of this class sorted by a field. + def all_sorted_by(field, order, options = {}) + search({ + size: 10_000, + query: { match_all: {} }, + sort: { field => { order: order } } + }.merge(options)) + end + + # Converts the model to an OpenStruct instance + # + # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil + # @return [OpenStruct] OpenStruct containing the selected fields + def to_os(*fields) + fields = all_fields if fields.empty? + OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) + end + + # Converts the model to a Hash + # + # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil + # @return [Hash] Hash containing the selected fields + def to_hsh(*fields) + fields = all_fields if fields.empty? + Hash[fields.map { |field| [field, send(field)] }] + end + +end
\ No newline at end of file diff --git a/app/repositories/category_repository.rb b/app/repositories/category_repository.rb new file mode 100644 index 0000000..e9cf033 --- /dev/null +++ b/app/repositories/category_repository.rb @@ -0,0 +1,21 @@ +require 'singleton' + +class CategoryRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "categories-#{Rails.env}" + + klass Category + + mapping do + indexes :id, type: 'keyword' + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/change_repository.rb b/app/repositories/change_repository.rb new file mode 100644 index 0000000..e5cc2f2 --- /dev/null +++ b/app/repositories/change_repository.rb @@ -0,0 +1,23 @@ +require 'singleton' + +class ChangeRepository < BaseRepository + include Singleton + + client ElasticsearchClient.default + + index_name "change-#{Rails.env}" + + klass Change + + mapping do + indexes :package, type: 'keyword' + indexes :category, type: 'keyword' + indexes :change_type, type: 'keyword' + indexes :version, type: 'keyword' + indexes :arches, type: 'keyword' + indexes :commit, type: 'object' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + +end diff --git a/app/repositories/elasticsearch_client.rb b/app/repositories/elasticsearch_client.rb new file mode 100644 index 0000000..88de0c8 --- /dev/null +++ b/app/repositories/elasticsearch_client.rb @@ -0,0 +1,13 @@ +class ElasticsearchClient + + def self.default + @default ||= Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' + end + + private + + def initialize(*) + raise "Should not be initialiazed" + end + +end
\ No newline at end of file diff --git a/app/repositories/package_repository.rb b/app/repositories/package_repository.rb new file mode 100644 index 0000000..ed77afb --- /dev/null +++ b/app/repositories/package_repository.rb @@ -0,0 +1,188 @@ +require 'forwardable' +require 'singleton' + +class PackageRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :suggest, :resolve, :find_atoms_by_useflag, :default_search_size, :default_search, + :build_query, :match_wildcard, :match_phrase, :match_description, :match_category, :scoring_functions + end + + index_name "packages-#{Rails.env}" + + klass Package + + mapping do + indexes :category, type: 'keyword' + indexes :name, type: 'keyword' + indexes :name_sort, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :description, type: 'text' + indexes :longdescription, type: 'text' + indexes :homepage, type: 'keyword' + indexes :license, type: 'keyword' + indexes :licenses, type: 'keyword' + indexes :herds, type: 'keyword' + indexes :maintainers, type: 'object' + indexes :useflags, type: 'object' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + def suggest(q) + PackageRepository.search( + size: 20, + query: { + wildcard: { + name_sort: { + wildcard: q.downcase + '*' + } + } + } + ) + end + + # Tries to resolve a query atom to one or more packages + def resolve(atom) + [] if atom.nil? || atom.empty? + + PackageRepository.find_all_by(:atom, atom) + PackageRepository.find_all_by(:name, atom) + end + + # Searches the versions index for versions using a certain USE flag. + # Results are aggregated by package atoms. + def find_atoms_by_useflag(useflag) + VersionRepository.search( + size: 0, # collect all packages. + query: { + bool: { + must: { match_all: {} }, + filter: { term: { use: useflag } } + } + }, + aggs: { + group_by_package: { + terms: { + field: 'package', + order: { '_key' => 'asc' }, + # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html + # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. + # Other limits in this app are also 10k mostly to 'make things fit kinda'. + size: 10000, + } + } + }, + ).response.aggregations['group_by_package'].buckets + end + + def default_search_size + 25 + end + + def default_search(q, offset) + return [] if q.nil? || q.empty? + + part1, part2 = q.split('/', 2) + + if part2.nil? + search(build_query(part1, nil, default_search_size, offset)) + else + search(build_query(part2, part1, default_search_size, offset)) + end + end + + def build_query(q, category, size, offset) + { + size: size, + from: offset, + query: { + function_score: { + query: { bool: bool_query_parts(q, category) }, + functions: scoring_functions + } + } + } + end + + def bool_query_parts(q, category = nil) + q_dwncsd = q.downcase + + query = { + must: [ + match_wildcard(q_dwncsd) + ], + should: [ + match_phrase(q_dwncsd), + match_description(q) + ] + } + + query[:must] << [match_category(category)] if category + + query + end + + def match_wildcard(q) + q = ('*' + q + '*') unless q.include? '*' + q.tr!(' ', '*') + + { + wildcard: { + name_sort: { + wildcard: q, + boost: 4 + } + } + } + end + + def match_phrase(q) + { + match_phrase: { + name: { + query: q, + boost: 5 + } + } + } + end + + def match_description(q) + { + match: { + description: { + query: q, + boost: 0.1 + } + } + } + end + + def match_category(cat) + { + match: { + category: { + query: cat, + boost: 2 + } + } + } + end + + def scoring_functions + [ + { + filter: { + term: { + category: 'virtual' + } + }, + weight: 0.6 + } + ] + end + +end diff --git a/app/repositories/useflag_repository.rb b/app/repositories/useflag_repository.rb new file mode 100644 index 0000000..26328f4 --- /dev/null +++ b/app/repositories/useflag_repository.rb @@ -0,0 +1,99 @@ +require 'singleton' + +class UseflagRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_flags, :suggest, :local_for, :global, :use_expand + end + + index_name "useflags-#{Rails.env}" + + klass Useflag + + mapping do + indexes :name, type: 'text' + indexes :description, type: 'text' + indexes :atom, type: 'keyword' + indexes :scope, type: 'keyword' + indexes :use_expand_prefix, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + + # Retrieves all flags sorted by their state + def get_flags(name) + result = { local: {}, global: [], use_expand: [] } + + find_all_by(:name, name).each do |flag| + case flag.scope + when 'local' + result[:local][flag.atom] = flag + when 'global' + result[:global] << flag + when 'use_expand' + result[:use_expand] << flag + end + end + + result + end + + def suggest(q) + results = search( + size: 20, + query: { match_phrase_prefix: { name: q } } + ) + + processed_results = {} + results.each do |result| + if processed_results.key? result.name + processed_results[result.name] = { + name: result.name, + description: '(multiple definitions)', + scope: 'multi' + } + else + processed_results[result.name] = result + end + end + + processed_results.values.sort { |a, b| a.name.length <=> b.name.length } + end + + # Loads the local USE flags for a given package in a name -> model hash + # + # @param [String] atom Package to find flags for + # @return [Hash] + def local_for(atom) + map_by_name find_all_by(:atom, atom) + end + + # Maps the global USE flags in the index by their name + # This is expensive! + # + def global + map_by_name find_all_by(:scope, 'global') + end + + # Maps the USE_EXPAND variables in the index by their name + # + def use_expand + map_by_name find_all_by(:scope, 'use_expand') + end + + private + + def map_by_name(collection) + map = {} + + collection.each do |item| + map[item.name] = item + end + + map + end + +end diff --git a/app/repositories/version_repository.rb b/app/repositories/version_repository.rb new file mode 100644 index 0000000..43168d2 --- /dev/null +++ b/app/repositories/version_repository.rb @@ -0,0 +1,56 @@ +require 'singleton' + +class VersionRepository < BaseRepository + include Singleton + + class << self + extend Forwardable + def_delegators :instance, :get_popular_useflags + end + + index_name "versions-#{Rails.env}" + + klass Version + + mapping do + indexes :version, type: 'keyword' + indexes :package, type: 'keyword' + indexes :atom, type: 'keyword' + indexes :sort_key, type: 'integer' + indexes :slot, type: 'keyword' + indexes :subslot, type: 'keyword' + indexes :eapi, type: 'keyword' + indexes :keywords, type: 'keyword' + indexes :masks do + indexes :arches, type: 'keyword' + indexes :atoms, type: 'keyword' + indexes :author, type: 'keyword' + indexes :date, type: 'keyword' + indexes :reason, type: 'text' + end + indexes :use, type: 'keyword' + indexes :restrict, type: 'keyword' + indexes :properties, type: 'keyword' + indexes :metadata_hash, type: 'keyword' + indexes :created_at, type: 'date' + indexes :updated_at, type: 'date' + end + + # Retrieves the most widely used USE flags by all versions + # Note that packages with many versions are over-represented + def get_popular_useflags(n = 50) + search( + query: { match_all: {} }, + aggs: { + group_by_flag: { + terms: { + field: 'use', + size: n + } + } + }, + size: 0 + ).response.aggregations['group_by_flag'].buckets + end + +end diff --git a/app/views/arches/keyworded.html.erb b/app/views/arches/keyworded.html.erb index b7ae03d..ae1df29 100644 --- a/app/views/arches/keyworded.html.erb +++ b/app/views/arches/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/arches/stable.html.erb b/app/views/arches/stable.html.erb index b1a4548..eb66245 100644 --- a/app/views/arches/stable.html.erb +++ b/app/views/arches/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@arch}-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/feeds/changes.atom.builder b/app/views/feeds/changes.atom.builder index 5991f45..a8af3df 100644 --- a/app/views/feeds/changes.atom.builder +++ b/app/views/feeds/changes.atom.builder @@ -10,7 +10,7 @@ atom_feed(id: atom_id(@feed_type, @feed_id, 'feed')) do |feed| @changes.each do |change| atom = cp_to_atom change.category, change.package - package = Package.find_by :atom, atom + package = PackageRepository.find_by :atom, atom if package.nil? logger.warn "Package for change (#{change}) nil!" next diff --git a/app/views/index/_package.html.erb b/app/views/index/_package.html.erb index eeb3109..a364209 100644 --- a/app/views/index/_package.html.erb +++ b/app/views/index/_package.html.erb @@ -1,4 +1,4 @@ -<%- package = Package.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> +<%- package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)); unless package.nil? -%> <tr> <td> <a href="<%= slf(package_path(cp_to_atom(change.category, change.package))) %>"> diff --git a/app/views/index/index.html.erb b/app/views/index/index.html.erb index 890a5f3..af86c9e 100644 --- a/app/views/index/index.html.erb +++ b/app/views/index/index.html.erb @@ -1,5 +1,5 @@ <div class="jumbotron"> - <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter Package.count %></span> Gentoo Packages</h2> + <h2 class="site-welcome stick-top">Welcome to the Home of <span class="text-primary"><%= number_with_delimiter PackageRepository.count %></span> Gentoo Packages</h2> <form action="<%= search_packages_path %>" method="get"> <div class="typeahead-container"> @@ -43,7 +43,7 @@ </div> <ul class="list-group"> <% @version_bumps.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'packages/changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/_metadata.html.erb b/app/views/packages/_metadata.html.erb index 426afd9..5568c08 100644 --- a/app/views/packages/_metadata.html.erb +++ b/app/views/packages/_metadata.html.erb @@ -3,7 +3,7 @@ <h3 class="panel-title"><%= t :box_metadata %></h3> </div> <ul class="list-group kk-metadata-list"> - <% if package.homepage.size > 1 %> + <% if !package.homepage.nil? && package.homepage.size > 1 %> <li class="kk-metadata-item list-group-item"> <div class="row"> <div class="col-xs-12 col-md-3 kk-metadata-key"> diff --git a/app/views/packages/_package_header.html.erb b/app/views/packages/_package_header.html.erb index 1b7876b..8c611da 100644 --- a/app/views/packages/_package_header.html.erb +++ b/app/views/packages/_package_header.html.erb @@ -25,7 +25,7 @@ <%= package.description %> </p> - <% unless package.homepage.empty? || package.homepage.first.nil? || package.homepage.first.empty? %> + <% unless package.homepage.nil? || package.homepage.first.nil? || package.homepage.first.empty? %> <p class="kk-package-homepage"> <%= content_tag :a, package.homepage.first, href: package.homepage.first, rel: 'nofollow' %> </p> diff --git a/app/views/packages/added.html.erb b/app/views/packages/added.html.erb index 97d5cb6..589226a 100644 --- a/app/views/packages/added.html.erb +++ b/app/views/packages/added.html.erb @@ -12,7 +12,7 @@ <% cache("added-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.latest_version } %> <% end %> </ul> diff --git a/app/views/packages/keyworded.html.erb b/app/views/packages/keyworded.html.erb index ff5b60c..a83a558 100644 --- a/app/views/packages/keyworded.html.erb +++ b/app/views/packages/keyworded.html.erb @@ -12,7 +12,7 @@ <% cache("keyworded-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/search.html.erb b/app/views/packages/search.html.erb index fe77dd3..c4b5a75 100644 --- a/app/views/packages/search.html.erb +++ b/app/views/packages/search.html.erb @@ -3,15 +3,15 @@ <% if @packages.size > 0 %> <div class="panel panel-default"> <div class="panel-heading"> - Results <%= @offset + 1 %>—<%= [@offset + Package.default_search_size, @packages.total].min %> of <%= @packages.total %> + Results <%= @offset + 1 %>—<%= [@offset + PackageRepository.default_search_size, @packages.total].min %> of <%= @packages.total %> </div> <div class="list-group"> <%= render partial: 'package_result_row', collection: @packages, as: 'package' %> </div> <div class="panel-footer"> <div class="btn-group" role="group" aria-label="Result navigation"> - <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - Package.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> - <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + Package.default_search_size), class: 'btn btn-default ' + ((@offset + Package.default_search_size) > @packages.total ? 'disabled' : '') %> + <%= link_to '< Prev', search_packages_path(q: params[:q], o: [@offset - PackageRepository.default_search_size, 0].max), class: 'btn btn-default' + (@offset > 0 ? '' : ' disabled') %> + <%= link_to 'Next >', search_packages_path(q: params[:q], o: @offset + PackageRepository.default_search_size), class: 'btn btn-default ' + ((@offset + PackageRepository.default_search_size) > @packages.total ? 'disabled' : '') %> </div> </div> </div> diff --git a/app/views/packages/stable.html.erb b/app/views/packages/stable.html.erb index 7b230fe..d9654de 100644 --- a/app/views/packages/stable.html.erb +++ b/app/views/packages/stable.html.erb @@ -12,7 +12,7 @@ <% cache("stable-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/packages/updated.html.erb b/app/views/packages/updated.html.erb index b774c58..af54ce1 100644 --- a/app/views/packages/updated.html.erb +++ b/app/views/packages/updated.html.erb @@ -12,7 +12,7 @@ <% cache("updated-full-#{@changes.hash}") do %> <ul class="list-group"> <% @changes.each do |change| - _package = Package.find_by(:atom, cp_to_atom(change.category, change.package)) %> + _package = PackageRepository.find_by(:atom, cp_to_atom(change.category, change.package)) %> <%= render partial: 'changed_package', object: change, as: 'change', locals: { package: _package, version: _package.version(change.version) } %> <% end %> </ul> diff --git a/app/views/useflags/_useflag_result_row.html.erb b/app/views/useflags/_useflag_result_row.html.erb index 084669f..3bdcd30 100644 --- a/app/views/useflags/_useflag_result_row.html.erb +++ b/app/views/useflags/_useflag_result_row.html.erb @@ -1,4 +1,4 @@ -<a class="list-group-item" href="<%= slf useflag_path useflag[:name] %>"> - <h3 class="kk-search-result-header"><%= useflag[:name] %></h3> - <%= useflag[:description] %> +<a class="list-group-item" href="<%= slf useflag_path useflag.name %>"> + <h3 class="kk-search-result-header"><%= useflag.name %></h3> + <%= useflag.description %> </a> diff --git a/config/initializers/elasticsearch.rb b/config/initializers/elasticsearch.rb index 4ced5b5..1037b1f 100644 --- a/config/initializers/elasticsearch.rb +++ b/config/initializers/elasticsearch.rb @@ -1,9 +1,10 @@ -require 'elasticsearch/persistence/model' +require 'elasticsearch/persistence' + +DEFAULT_CLIENT = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' -Elasticsearch::Persistence.client = Elasticsearch::Client.new host: ENV['ELASTICSEARCH_URL'] || 'localhost:9200' if Rails.env.development? or ENV['RAILS_DEBUG'] logger = ActiveSupport::Logger.new(STDERR) logger.level = Logger::DEBUG logger.formatter = proc { |s, d, p, m| "\e[2m#{m}\n\e[0m" } - Elasticsearch::Persistence.client.transport.logger = logger + DEFAULT_CLIENT.transport.logger = logger end diff --git a/lib/kkuleomi/store.rb b/lib/kkuleomi/store.rb index a1a2d93..a36f0ca 100644 --- a/lib/kkuleomi/store.rb +++ b/lib/kkuleomi/store.rb @@ -1,15 +1,12 @@ module Kkuleomi::Store - def self.refresh_index - Category.gateway.refresh_index! - end def self.create_index(force = false) - types = [ - Category, - Package, - Version, - Change, - Useflag, + repositories = [ + CategoryRepository, + PackageRepository, + VersionRepository, + ChangeRepository, + UseflagRepository, ] base_settings = { @@ -33,15 +30,11 @@ module Kkuleomi::Store mapping: { total_fields: { limit: 50000 } } } + settings = JSON.parse('{ "mapping": { "total_fields": { "limit": 50000 } } }') + # In ES 1.5, we could use 1 mega-index. But in ES6, each model needs its own. - types.each { |type| - client = type.gateway.client - client.indices.delete(index: type.index_name) rescue nil if force - body = { - settings: type.settings.to_hash.merge(base_settings), - mappings: type.mappings.to_hash - } - client.indices.create(index: type.index_name, body: body) + repositories.each { |repository| + repository.instance.create_index!(force: true, settings: settings) } end end diff --git a/lib/kkuleomi/store/model.rb b/lib/kkuleomi/store/model.rb deleted file mode 100644 index 653884b..0000000 --- a/lib/kkuleomi/store/model.rb +++ /dev/null @@ -1,78 +0,0 @@ -module Kkuleomi::Store::Model - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - # Finds instances by exact IDs using the 'term' filter - def find_all_by(field, value, opts = {}) - search({ - size: 10_000, - query: { bool: { filter: { term: { field => value } } } } - }.merge(opts)) - end - - # Filter all instances by the given parameters - def filter_all(filters, opts = {}) - filter_args = [] - filters.each_pair { |field, value| filter_args << { term: { field => value } } } - - search({ - query: { - bool: { filter: { bool: { must: filter_args } } } - }, - size: 10_000 - }.merge(opts)) - end - - def find_by(field, value, opts = {}) - find_all_by(field, value, opts).first - end - - def find_all_by_parent(parent, opts = {}) - search(opts.merge( - size: 10_000, - query: { - bool: { - filter: { - has_parent: { - parent_type: parent.class.document_type, - query: { term: { _id: parent.id } } - } - }, - must: { match_all: {} } - } - } - )) - end - - # Returns all (by default 10k) records of this class sorted by a field. - def all_sorted_by(field, order, options = {}) - all({ - query: { match_all: {} }, - sort: { field => { order: order } } - }, options) - end - end - - module InstanceMethods - # Converts the model to an OpenStruct instance - # - # @param [Array<Symbol>] fields Fields to export into the OpenStruct, or all fields if nil - # @return [OpenStruct] OpenStruct containing the selected fields - def to_os(*fields) - fields = all_fields if fields.empty? - OpenStruct.new(Hash[fields.map { |field| [field, send(field)] }]) - end - - # Converts the model to a Hash - # - # @param [Array<Symbol>] fields Fields to export into the Hash, or all fields if nil - # @return [Hash] Hash containing the selected fields - def to_hsh(*fields) - fields = all_fields if fields.empty? - Hash[fields.map { |field| [field, send(field)] }] - end - end -end diff --git a/lib/kkuleomi/store/models/package_import.rb b/lib/kkuleomi/store/models/package_import.rb index 99ab433..8ae1e5d 100644 --- a/lib/kkuleomi/store/models/package_import.rb +++ b/lib/kkuleomi/store/models/package_import.rb @@ -30,15 +30,15 @@ module Kkuleomi::Store::Models::PackageImport set_basic_metadata(package_model, latest_ebuild) # Be sure to have an ID now - save + PackageRepository.save(self) import_useflags!(package_model) - Kkuleomi::Store.refresh_index + CategoryRepository.refresh_index! import_versions!(package_model, ebuilds, options) # Do this last, so that any exceptions before this point skip this step self.metadata_hash = package_model.metadata_hash - save + PackageRepository.save(self) if options[:package_state] == 'new' && !options[:suppress_change_objects] RecordChangeJob.perform_later( @@ -73,7 +73,7 @@ module Kkuleomi::Store::Models::PackageImport end def import_useflags!(package_model) - index_flags = Useflag.local_for(package_model.to_cp) + index_flags = UseflagRepository.local_for(package_model.to_cp) model_flags = package_model.metadata[:use] new_flags = model_flags.keys - index_flags.keys @@ -87,23 +87,23 @@ module Kkuleomi::Store::Models::PackageImport flag_doc.description = model_flags[flag] flag_doc.atom = package_model.to_cp flag_doc.scope = 'local' - flag_doc.save + UseflagRepository.save(flag_doc) end eql_flags.each do |flag| unless index_flags[flag].description == model_flags[flag] index_flags[flag].description = model_flags[flag] - index_flags[flag].save + UseflagRepository.save(index_flags[flag]) end end del_flags.each do |flag| - index_flags[flag].delete + UseflagRepository.delete(index_flags[flag]) end end def import_versions!(package_model, ebuilds, options) - index_v = Hash[Version.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] + index_v = Hash[VersionRepository.find_all_by(:package, package_model.to_cp).map { |v| [v.version, v] }] model_v = Hash[ebuilds.map { |v| [v.version, v] }] index_keys = index_v.keys @@ -128,7 +128,7 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end @@ -144,12 +144,12 @@ module Kkuleomi::Store::Models::PackageImport if sort_key == 0 self.useflags = version_doc.useflags - save + VersionRepository.save(version_doc) end end del_v.each do |v| - index_v[v].delete + VersionRepository.delete(index_v[v]) end end end diff --git a/lib/kkuleomi/store/models/package_search.rb b/lib/kkuleomi/store/models/package_search.rb deleted file mode 100644 index ec0268c..0000000 --- a/lib/kkuleomi/store/models/package_search.rb +++ /dev/null @@ -1,161 +0,0 @@ -# Contains the search logic for packages -module Kkuleomi::Store::Models::PackageSearch - def self.included(base) - base.send :include, InstanceMethods - base.extend ClassMethods - end - - module ClassMethods - def suggest(q) - Package.search( - size: 20, - query: { - wildcard: { - name_sort: { - wildcard: q.downcase + '*' - } - } - } - ) - end - - # Tries to resolve a query atom to one or more packages - def resolve(atom) - [] if atom.nil? || atom.empty? - - Package.find_all_by(:atom, atom) + Package.find_all_by(:name, atom) - end - - # Searches the versions index for versions using a certain USE flag. - # Results are aggregated by package atoms. - def find_atoms_by_useflag(useflag) - Version.search( - size: 0, # collect all packages. - query: { - bool: { - must: { match_all: {} }, - filter: { term: { use: useflag } } - } - }, - aggs: { - group_by_package: { - terms: { - field: 'package', - order: { '_key' => 'asc' }, - # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-terms-aggregation.html - # ES actually dislikes large sizes like this (it defines 10k buckets basically) and it will be *very* expensive but lets try it and see. - # Other limits in this app are also 10k mostly to 'make things fit kinda'. - size: 10000, - } - } - }, - ).response.aggregations['group_by_package'].buckets - end - - def default_search_size - 25 - end - - def default_search(q, offset) - return [] if q.nil? || q.empty? - - part1, part2 = q.split('/', 2) - - if part2.nil? - search(build_query(part1, nil, default_search_size, offset)) - else - search(build_query(part2, part1, default_search_size, offset)) - end - end - - def build_query(q, category, size, offset) - { - size: size, - from: offset, - query: { - function_score: { - query: { bool: bool_query_parts(q, category) }, - functions: scoring_functions - } - } - } - end - - def bool_query_parts(q, category = nil) - q_dwncsd = q.downcase - - query = { - must: [ - match_wildcard(q_dwncsd) - ], - should: [ - match_phrase(q_dwncsd), - match_description(q) - ] - } - - query[:must] << [match_category(category)] if category - - query - end - - def match_wildcard(q) - q = ('*' + q + '*') unless q.include? '*' - q.tr!(' ', '*') - - { - wildcard: { - name_sort: { - wildcard: q, - boost: 4 - } - } - } - end - - def match_phrase(q) - { - match_phrase: { - name: { - query: q, - boost: 5 - } - } - } - end - - def match_description(q) - { - match: { - description: { - query: q, - boost: 0.1 - } - } - } - end - - def match_category(cat) - { - match: { - category: { - query: cat, - boost: 2 - } - } - } - end - - def scoring_functions - [ - { - filter: { term: { category: 'virtual' } }, - weight: 0.6 - } - ] - end - end - - module InstanceMethods - end -end diff --git a/lib/kkuleomi/store/models/version_import.rb b/lib/kkuleomi/store/models/version_import.rb index b65b683..6ee6b64 100644 --- a/lib/kkuleomi/store/models/version_import.rb +++ b/lib/kkuleomi/store/models/version_import.rb @@ -38,7 +38,7 @@ module Kkuleomi::Store::Models::VersionImport self.masks = Portage::Util::Masks.for(ebuild_model) self.metadata_hash = ebuild_model.metadata_hash - save() + VersionRepository.save(self) # If keywords changed, calculate changes and record as needed (but only do that if we should) unless options[:suppress_change_objects] @@ -60,7 +60,7 @@ module Kkuleomi::Store::Models::VersionImport # @param [Package] parent Parent package model def set_sort_key!(key, parent) self.sort_key = key - save() + VersionRepository.save(self) end def strip_useflag_defaults(flags) |