Prefill original title and content from projecteuler.net

Gemfile

@@ -23,6 +23,8 @@ gem 'rails-i18n', '~> 6.0.0'
 
 gem 'rails-controller-testing'
 
+gem 'webmock', group: :test
+
 # Use jquery as the JavaScript library
 gem 'jquery-rails', '~> 4.3.5'
 # Turbolinks makes following links in your web application faster. Read more: https://github.com/rails/turbolinks

Gemfile.lock

@@ -56,6 +56,8 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2)
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
     autoprefixer-rails (9.7.3)
       execjs
     bcrypt (3.1.13)
@@ -80,6 +82,8 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     concurrent-ruby (1.1.6)
+    crack (0.4.3)
+      safe_yaml (~> 1.0.0)
     crass (1.0.6)
     devise (4.7.1)
       bcrypt (~> 3.0)
@@ -99,6 +103,7 @@ GEM
       sassc (>= 1.11)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
+    hashdiff (1.0.1)
     hashie (4.1.0)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
@@ -146,6 +151,7 @@ GEM
       omniauth (~> 1.9)
     orm_adapter (0.5.0)
     popper_js (1.14.5)
+    public_suffix (4.0.4)
     rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -187,6 +193,7 @@ GEM
     responders (3.0.0)
       actionpack (>= 5.0)
       railties (>= 5.0)
+    safe_yaml (1.0.5)
     sassc (2.2.1)
       ffi (~> 1.9)
     sassc-rails (2.1.2)
@@ -226,6 +233,10 @@ GEM
       activemodel (>= 6.0.0)
       bindex (>= 0.4.0)
       railties (>= 6.0.0)
+    webmock (3.8.3)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
@@ -262,8 +273,9 @@ DEPENDENCIES
   tzinfo-data
   uglifier (~> 4.2.0)
   web-console (~> 4.0.1)
+  webmock
   will_paginate (~> 3.3.0)
   will_paginate-bootstrap4 (~> 0.2.2)
 
 BUNDLED WITH
-   2.0.2
+   2.1.4

app/controllers/admin/dashboard_controller.rb

@@ -1,16 +1,11 @@
 class Admin::DashboardController < AdminController
   def index
     @current_problem_count = Problem.count
+    @most_recent_pull = Problem.maximum(:pulled_at)
   end
 
-  def update_problem_count
+  def pull_problems
-    begin
+    PullProblemsJob.perform_later
-      new_problem_count = params[:problem_count].to_i
+    redirect_to({:controller => 'admin/dashboard', :action => :index}, notice: t('.pull_problems_initiated'))
-      raise t('no_problem_count') unless new_problem_count
-      Problem.update_count(new_problem_count)
-      redirect_to({:controller => 'admin/dashboard', :action => :index}, notice: t('.success_message'))
-    rescue => e
-      redirect_to({:controller => 'admin/dashboard', :action => :index}, alert: t('.failure_message', error: e.message))
-    end
   end
 end

app/controllers/problems_controller.rb

@@ -7,9 +7,6 @@ class ProblemsController < ApplicationController
   end
 
   def show
-    unless @problem.is_translated?
-      render action: "untranslated"
-    end
   end
 
   private

app/controllers/translations_controller.rb

@@ -8,6 +8,9 @@ class TranslationsController < ApplicationController
     if @problem.is_translated?
       @translation.title = @problem.translation.title
       @translation.content = @problem.translation.content
+    else
+      @translation.title = @problem.original_title
+      @translation.content = @problem.original_content
     end
   end
 

app/jobs/pull_problem_content_job.rb

@@ -0,0 +1,21 @@
+require 'open-uri'
+
+class PullProblemContentJob < ApplicationJob
+  queue_as :default
+
+  def perform(problem)
+    html = URI.open("https://projecteuler.net/minimal=#{problem.id}").read
+    html.strip!
+
+    # Linked problems
+    html.gsub!('<a href="problem=', '<a href="/problem=')
+    # Linked about pages
+    html.gsub!('<a href="about=', '<a href="/about=')
+    # Linked txt resources
+    html.gsub!('<a href="project/resources/', '<a href="https://projecteuler.net/project/resources/')
+    # Included images
+    html.gsub!('<img src="project/images/', '<img src="https://projecteuler.net/project/images/')
+
+    problem.update(original_content: html, pulled_at: Time.current())
+  end
+end

app/jobs/pull_problems_job.rb

@@ -0,0 +1,23 @@
+require 'csv'
+require 'open-uri'
+
+class PullProblemsJob < ApplicationJob
+  queue_as :default
+
+  PULL_URL = "https://projecteuler.net/minimal=problems;csv"
+
+  def perform
+    csv = CSV.parse(URI.open(PULL_URL), headers: [:id, :title, :date_published, :date_last_updated, :solved_by])
+    csv.each do |row|
+      id = row[:id].to_i
+      last_updated = Time.at(row[:date_last_updated].to_i)
+      problem = Problem.where(id: id).first_or_create!
+      if problem.pulled_at.nil? or problem.pulled_at < last_updated
+        original_title = row[:title]
+        problem.update(original_title: original_title)
+        # Don't update pulled_at yet until we also successfully pulled the original_content
+        PullProblemContentJob.perform_later problem
+      end
+    end
+  end
+end

app/views/admin/dashboard/index.html.erb

@@ -3,9 +3,10 @@
   <h1><%= t('.administration') %></h1>
 </div>
 <%= link_to t('.view_translations'), admin_translations_path, class: 'btn btn-primary' %>
-
+<p>
-<h1><%= t('.update_problem_count') %></h1>
+  <%= t('.number_of_problems') %>: <%= @current_problem_count %><br/>
-<%= form_tag '/admin/update_problem_count', method: :post, class: 'form-inline' do %>
+  <% if @most_recent_pull.present? %>
-    <%= number_field_tag 'problem_count', @current_problem_count, min: @current_problem_count, class: 'form-control' %>
+    <%= t('.time_since_last_pull') %>: <%= time_ago_in_words @most_recent_pull %>
-    <%= submit_tag t('.update'), class: 'btn btn-warning' %>
+  <% end %>
-<% end %>
+</p>
+<%= button_to t('.pull_problems'), '/admin/pull_problems', method: :post, class: 'btn btn-warning' %>

app/views/problems/index.html.erb

@@ -28,12 +28,10 @@
           <% if problem.is_translated? %>
             <%= link_to problem.title, problem %>
           <% else %>
-              <i><%= t 'problems.not_yet_translated' %></i>
+            <%= link_to problem do %>
-              <%= link_to new_problem_translation_path(problem), class: 'btn btn-primary btn-sm' do %>
+              <i><%= problem.original_title %> <%= t 'problems.not_translated_yet' %></i>
-                  <%= icon('fas', 'edit') %> <%= t '.suggest_translation' %>
             <% end %>
           <% end %>
-
         </td>
       </tr>
   <% end %>

app/views/problems/show.html.erb

@@ -1,7 +1,13 @@
 <% provide(:title, t('problems.show.problem_subtitle', id: @problem.id)) %>
 
 <div class="pb-2 mt-4 mb-2">
-  <h1><%= @problem.title %></h1>
+  <h1>
+    <% if @problem.is_translated? %>
+      <%= @problem.title %>
+    <% else %>
+      <i><%= @problem.original_title %> <%= t 'problems.not_translated_yet' %></i>
+    <% end %>
+  </h1>
 </div>
 <% if Problem.exists?(@problem.id-1) %>
   <%= link_to problem_path(@problem.id-1), title: t('problems.show.problem_subtitle', id: @problem.id-1), class: 'problem-prev' do %>
@@ -19,15 +25,26 @@
   </div>
   <div class="problem-buttons">
     <%= link_to new_problem_translation_path(@problem), class: 'problem-buttons-inner btn btn-primary btn-sm' do %>
-      <%= icon('fas', 'edit') %> <%= t '.improve_translation' %>
+      <%= icon('fas', 'edit') %>
+      <% if @problem.is_translated? %>
+        <%= t '.improve_translation' %>
+      <% else %>
+        <%= t '.suggest_translation' %>
+      <% end %>
     <% end %>
   </div>
   <div class="card-body problem-content">
+    <% if @problem.is_translated? %>
       <%= sanitize @problem.content, scrubber: TranslationContentScrubber.new %>
+    <% else %>
+      <%= sanitize @problem.original_content, scrubber: TranslationContentScrubber.new %>
+    <% end %>
   </div>
+  <% if @problem.is_translated? %>
     <div class="card-footer text-muted">
       <%= render 'shared/authors', authors: @problem.authors, has_anonymous_author: @problem.has_anonymous_author? %>
     </div>
+  <% end %>
 </div>
 <div class="text-center">
   <%= link_to t('.view_original_problem'), @problem.original_url, target: '_blank' %>

app/views/problems/untranslated.html.erb

@@ -1,10 +0,0 @@
-<% provide(:title, t('problems.show.problem_subtitle', id: @problem.id)) %>
-
-<div class="pb-2 mt-4 mb-2 border-bottom">
-  <h1><%= t 'problems.show.problem_subtitle', id: @problem.id %></h1>
-</div>
-
-<%= t 'problems.not_yet_translated' %>
-<%= link_to new_problem_translation_path(@problem), class: 'btn btn-primary btn-sm' do %>
-    <%= icon('fas', 'edit') %> <%= t 'problems.index.suggest_translation' %>
-<% end %>

app/views/translations/_translation-tips.de.html.erb

@@ -1,15 +1,12 @@
-<div class="card">
+<div class="card mb-3">
   <div class="card-header">
     Hinweise zur Erstellung von Übersetzungen
   </div>
   <div class='card-body'>
     <ul>
-      <li>Als Basis für jede Übersetzung dient der HTML-Quelltext von projecteuler.net.
+      <li>Als Basis für jede Übersetzung dient der <%= link_to "Text von projecteuler.net", @problem.original_url, target: '_blank' %>.</li>
-        Dazu gehen Sie auf die entsprechende <%= link_to "Problem-Seite", @problem.original_url, target: '_blank' %>.
+      <li>Der Originaltitel und Text im untenstehenden Formular sind zu übersetzen.</li>
-        Dort öffnen Sie den HTML-Quelltext (z.B. mit <code>Strg+U</code> in Firefox) und kopieren alles zwischen <code>&lt;div class=&quot;problem_content&quot; role=&quot;problem&quot;&gt;</code> und <code>&lt;/div&gt;</code>.
+      <li>Der HTML-Quelltext in seiner Struktur sollte im Allgemeinen unverändert bleiben.</li>
-        Diesen Quelltext fügen Sie auf dieser Seite ein.
-        In Zukunft sollen diese Schritte automatisch passieren.</li>
-      <li>Der HTML-Quelltext sollte im Allgemeinen unverändert bleiben.</li>
       <li>Querverweise zu anderen Problemen sollten in der Form <code>/problem=??</code> erfolgen.</li>
       <li>Wenn Grafiken oder Dateien verwendet werden, sollte die URL <code>https://projecteuler.net/...</code> verwendet werden.</li>
       <li>Die Sie-Form benutzen.</li>

config/locales/views/admin/de.yml

@@ -5,12 +5,11 @@ de:
         administration: "Administration"
         translations: "Übersetzungen"
         view_translations: "Übersetzungen anschauen"
-        update_problem_count: "Problem-Anzahl aktualisieren"
+        pull_problems: "Probleme aktualisieren"
-        update: "Aktualisieren"
+        number_of_problems: "Anzahl Probleme"
-      update_problem_count:
+        time_since_last_pull: "Zeit seit letzter erfolgreicher Aktualisierung"
-        success_message: "Problem-Anzahl wurde erfolgreich aktualisiert!"
+      pull_problems:
-        failure_message: "Problem-Anzahl konnte nicht aktualisiert werden! Grund: %{error}"
+        pull_problems_initiated: "Das Aktualisieren der Probleme wurde angestoßen."
-        no_problem_count: "Keine Problem-Anzahl gegeben!"
     translations:
       index:
         anonymous: "Anonym"

config/locales/views/problems/de.yml

@@ -2,11 +2,11 @@
 
 de:
   problems:
-    not_yet_translated: Dieses Problem wurde noch nicht übersetzt.
+    not_translated_yet: "(noch nicht übersetzt)"
     index:
-      suggest_translation: Übersetzung vorschlagen
       translated_count: "Übersetzte Probleme: %{translated} von %{total}"
     show:
       improve_translation: Übersetzung verbessern
       problem_subtitle: Problem %{id}
       view_original_problem: Dieses Problem auf projecteuler.net
+      suggest_translation: Übersetzung vorschlagen

config/routes.rb

@@ -32,7 +32,7 @@ Rails.application.routes.draw do
 
   namespace :admin do
     get '', to: 'dashboard#index', as: 'dashboard_index'
-    post '/update_problem_count', to: 'dashboard#update_problem_count', as: 'dashboard_update_problem_count'
+    post '/pull_problems', to: 'dashboard#pull_problems', as: 'dashboard_pull_problems'
     resources :translations, only: [:index] do
       get '', to: 'translations#show', as: ''
       post 'accept', to: 'translations#accept'

db/migrate/20200501095104_add_original_data_to_problems.rb

@@ -0,0 +1,7 @@
+class AddOriginalDataToProblems < ActiveRecord::Migration[6.0]
+  def change
+    add_column :problems, :pulled_at, :datetime
+    add_column :problems, :original_title, :string
+    add_column :problems, :original_content, :text
+  end
+end

db/schema.rb

@@ -2,20 +2,23 @@
 # of editing this file, please use the migrations feature of Active Record to
 # incrementally modify your database, and then regenerate this schema definition.
 #
-# Note that this schema.rb definition is the authoritative source for your
+# This file is the source Rails uses to define your schema when running `rails
-# database schema. If you need to create the application database on another
+# db:schema:load`. When creating a new database, `rails db:schema:load` tends to
-# system, you should be using db:schema:load, not running all the migrations
+# be faster and is potentially less error prone than running all of your
-# from scratch. The latter is a flawed and unsustainable approach (the more migrations
+# migrations from scratch. Old migrations may fail to apply correctly if those
-# you'll amass, the slower it'll run and the greater likelihood for issues).
+# migrations use external dependencies or application code.
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_28_084343) do
+ActiveRecord::Schema.define(version: 2020_05_01_095104) do
 
   create_table "problems", force: :cascade do |t|
     t.datetime "created_at"
     t.datetime "updated_at"
     t.integer "translation_id"
+    t.datetime "pulled_at"
+    t.string "original_title"
+    t.text "original_content"
     t.index ["translation_id"], name: "index_problems_on_translation_id"
   end
 

test/controllers/admin/dashboard_controller_test.rb

@@ -2,6 +2,7 @@ require 'test_helper'
 
 class Admin::DashboardControllerTest < ActionDispatch::IntegrationTest
   include Devise::Test::IntegrationHelpers
+  include ActiveJob::TestHelper
 
   setup do
   end
@@ -18,18 +19,14 @@ class Admin::DashboardControllerTest < ActionDispatch::IntegrationTest
     assert_response :success
   end
 
-  test "should post new problem count" do
+  test "should update problems" do
-    login_admin
+    WebMock.stub_request(:get, "https://projecteuler.net/minimal=problems;csv").to_return(body: "")
-    post admin_dashboard_update_problem_count_url(problem_count: 15)
-    assert_redirected_to admin_dashboard_index_url
-    assert_equal 15, Problem.count
-  end
 
-  test "should fail incorrect problem count" do
     login_admin
-    post admin_dashboard_update_problem_count_url(problem_count: 3)
+    assert_enqueued_jobs 1, only: PullProblemsJob do
+      post admin_dashboard_pull_problems_url
+    end
     assert_redirected_to admin_dashboard_index_url
-    assert_equal 4, Problem.count
   end
 
 end

test/jobs/pull_problem_content_job_test.rb

@@ -0,0 +1,15 @@
+require 'test_helper'
+
+class PullProblemContentJobTest < ActiveJob::TestCase
+  test "should update problem count and titles" do
+
+    stub = WebMock.stub_request(:get, "https://projecteuler.net/minimal=3").
+        to_return(body: "\n<p><a href=\"problem=5\">test</a></p>")
+
+    PullProblemContentJob.perform_now problems(:three)
+
+    assert_requested stub
+    assert_equal '<p><a href="/problem=5">test</a></p>', Problem.find(3).original_content
+    assert_in_delta Time.current, Problem.find(3).pulled_at, 5
+  end
+end

test/jobs/pull_problems_job_test.rb

@@ -0,0 +1,26 @@
+require 'test_helper'
+require 'csv'
+
+class PullProblemsJobTest < ActiveJob::TestCase
+  test "should update problem count and titles" do
+    csv_string = CSV.generate do |csv|
+      csv << [1, "Title of problem 1", 4.days.ago.to_i, 2.days.ago.to_i, 123]
+      csv << [2, "Title of problem 2", 4.days.ago.to_i, 2.days.ago.to_i, 123]
+      csv << [3, "Title of problem 3", 4.days.ago.to_i, 2.days.ago.to_i, 123]
+      csv << [4, "Title of problem 4", 4.days.ago.to_i, 2.days.ago.to_i, 123]
+      csv << [5, "Title of problem 5", 4.days.ago.to_i, 2.days.ago.to_i, 123]
+    end
+    stub = WebMock.stub_request(:get, "https://projecteuler.net/minimal=problems;csv").
+        to_return(body: csv_string)
+    problems(:two).update(pulled_at: 3.days.ago)
+    problems(:three).update(pulled_at: 1.day.ago)
+
+    assert_enqueued_jobs(4, only: PullProblemContentJob) do
+      PullProblemsJob.perform_now
+    end
+
+    assert_requested stub
+    assert_equal 5, Problem.count
+    assert_equal "Title of problem 2", Problem.find(2).original_title
+  end
+end

test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'codacy-coverage'
+require 'webmock/minitest'
 
 Codacy::Reporter.start
 
@@ -10,6 +11,8 @@ class ActiveSupport::TestCase
   # Setup all fixtures in test/fixtures/*.yml for all tests in alphabetical order.
   fixtures :all
 
+  WebMock.disable_net_connect!
+
   # Add more helper methods to be used by all tests here...
 end