Import xls files

.gitignore

@@ -21,6 +21,7 @@ Gemfile.ci.lock
 
 public/avatars/**/*
 public/assets
+public/importers/
 
 tmp
 Design
@@ -37,3 +38,6 @@ Design
 
 .passenger
 .vagrant
+
+docker-compose.override.yml
+.local/

Dockerfile

@@ -20,7 +20,8 @@ RUN apt-get update && \
 	apt-get autoremove -y && \
 	cp config/database.postgres.docker.yml config/database.yml && \
 	gem install bundler && \
-	bundle install --deployment && \
+	bundle config set deployment 'true' && \
+	bundle install && \
 	bundle exec rails assets:precompile
 
 CMD ["bundle","exec","rails","s"]

Gemfile

@@ -100,3 +100,8 @@ gem "devise-encryptable"
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
 gem 'activejob', '~> 5.2.0'
 gem 'ransack_ui', path: 'vendor/gems/ransack_ui-1.3.4' # Vendored until our fix is merged and released
+gem 'spreadsheet'
+
+gem "roo", "~> 2.8"
+
+gem "roo-xls", "~> 1.2"

Gemfile.lock

@@ -176,6 +176,7 @@ GEM
     jquery-ui-rails (6.0.1)
       railties (>= 3.2.16)
     libv8 (3.16.14.19)
+    libv8 (3.16.14.19-x86_64-linux)
     listen (3.2.1)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
@@ -302,6 +303,13 @@ GEM
       railties (>= 4.2.0, < 6.0)
     responds_to_parent (2.0.0)
       actionpack (>= 3.2.22, < 6.0)
+    roo (2.8.3)
+      nokogiri (~> 1)
+      rubyzip (>= 1.3.0, < 3.0.0)
+    roo-xls (1.2.0)
+      nokogiri
+      roo (>= 2.0.0, < 3)
+      spreadsheet (> 0.9.0)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -334,6 +342,7 @@ GEM
       rainbow (>= 2.2.2, < 4.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-ole (1.2.12.2)
     ruby-progressbar (1.10.1)
     rubyzip (2.3.0)
     sass (3.7.4)
@@ -359,6 +368,8 @@ GEM
     sixarm_ruby_unaccent (1.2.0)
     sort_alphabetical (1.1.0)
       unicode_utils (>= 1.2.2)
+    spreadsheet (1.2.6)
+      ruby-ole (>= 1.0)
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
@@ -403,6 +414,7 @@ GEM
 
 PLATFORMS
   ruby
+  x86_64-linux
 
 DEPENDENCIES
   activejob (~> 5.2.0)
@@ -462,6 +474,8 @@ DEPENDENCIES
   rb-inotify
   responders (~> 2.0)
   responds_to_parent
+  roo (~> 2.8)
+  roo-xls (~> 1.2)
   rspec-activemodel-mocks
   rspec-rails
   rubocop (~> 0.76.0)
@@ -470,6 +484,7 @@ DEPENDENCIES
   select2-rails
   selenium-webdriver
   simple_form
+  spreadsheet
   sprockets-rails (>= 3.0.0)
   sqlite3 (~> 1.3.13)
   therubyracer
@@ -482,4 +497,4 @@ DEPENDENCIES
   zeus
 
 BUNDLED WITH
-   2.1.4
+   2.2.2

app/controllers/importers_controller.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2008-2013 Michael Dvorkin and contributors.
+#
+# Fat Free CRM is freely distributable under the terms of MIT license.
+# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php
+#------------------------------------------------------------------------------
+
+require 'json'
+
+class ImportersController < ApplicationController
+  # get /importers/new                                                 AJAX
+  #----------------------------------------------------------------------------
+  def new
+    @importer = Importer.new
+    @importer.entity_type = params[:entity_type]
+    @importer.entity_id = params[:entity_id] if params[:entity_id]
+    respond_with(@importer)
+  end
+
+  # post /importers/create
+  #----------------------------------------------------------------------------
+  def create
+    errors = false
+    if params[:importer]
+      @importer = Importer.create(importer_params)
+      if @importer.valid?
+        @importer.save
+      else
+        errors = @importer.errors.full_messages
+      end
+    end
+
+    respond_to do |format|
+      if errors
+        format.html { render "create", locals: { errors: errors } }
+      else
+        format.html { redirect_to form_map_columns_importer_path(@importer) }
+      end
+    end
+  end
+
+  # get /importers/:id/map
+  #----------------------------------------------------------------------------
+  def form_map_columns
+    @importer = Importer.find(params[:id])
+    columns = FatFreeCRM::ImportHandle.get_columns(@importer.attachment.path)
+
+    attributes = []
+    attributes_extra = []
+
+    object = @importer.entity_class
+    attrs = object.attribute_names - ['id']
+
+    attrs.each do |attr|
+      attributes.push(
+        name: attr,
+          required: object.validators_on(attr).any? { |v| v.is_a? ActiveModel::Validations::PresenceValidator }
+      )
+    end
+
+    if @importer.entity_type == 'lead'
+      attrs = Address.attribute_names - %w[id created_at updated_at deleted_at address_type addressable_type addressable_id]
+
+      attrs.each do |attr|
+        attributes_extra.push(
+          name: attr,
+            required: Address.validators_on(attr).any? { |v| v.is_a? ActiveModel::Validations::PresenceValidator }
+        )
+      end
+    end
+
+    respond_to do |format|
+      format.html { render "form_map_columns", locals: { columns: columns, attributes: attributes, attributes_extra: attributes_extra } }
+    end
+  end
+
+  # post /importers/:id/map
+  #----------------------------------------------------------------------------
+  def map_columns
+    @importer = Importer.find(params[:id])
+    @importer.status = :map
+    map = params[:map]
+    @importer.map = map.to_json
+    @importer.save
+    @importer = FatFreeCRM::ImportHandle.process(@importer)
+
+    respond_to do |format|
+      format.html { render "map_columns" }
+    end
+  end
+
+  #   # get /campaigns/import                                                 AJAX
+  #   #----------------------------------------------------------------------------
+  #   def import
+  #     @importer = Importer.new
+  #     @importer.entity_type = 'Campaign'
+  #     respond_with(@importer)
+  #   end
+  #
+  #   # patch /campaigns/import                                                 AJAX
+  #   #----------------------------------------------------------------------------
+  #   def import_upload
+  #     @error = false
+  #     @result = {
+  #         items: [],
+  #         errors: []
+  #     }
+  #
+  #     if params[:importer]
+  #       @importer = Importer.create(import_params)
+  #       if @importer.valid?
+  #         @importer.save
+  #         @result = FatFreeCRM::ImportHandle.process(@importer)
+  #       else
+  #         puts @importer.errors.full_messages
+  #         @result[:errors].push(@importer.errors.full_messages)
+  #         @error = true
+  #       end
+  #     end
+  #     respond_with(@error,@result)
+  #   end
+  #
+  #
+  #   # get /campaigns/%id/import                                                 AJAX
+  #   #----------------------------------------------------------------------------
+  #   def import_leads
+  #     @importer = Importer.new
+  #     @importer.entity_type = 'Lead'
+  #     respond_with(@importer)
+  #   end
+  #
+  #   # patch /campaigns/import                                                 AJAX
+  #   #----------------------------------------------------------------------------
+  #   def uploads_import_leads
+  #     @error = false
+  #     @result = {
+  #         items: [],
+  #         errors: []
+  #     }
+  #
+  #     if params[:importer]
+  #       @importer = Importer.create(import_params)
+  #       if @importer.valid?
+  #         @importer.save
+  #         @colummns = FatFreeCRM::ImportHandle.get_columns(@importer.attachment.path)
+  #       else
+  #         puts @importer.errors.full_messages
+  #         @result[:errors].push(@importer.errors.full_messages)
+  #         @error = true
+  #       end
+  #     end
+  #     respond_with(@colummns) do |format|
+  #       format.js { render :uploads_import_leads }
+  #     end
+  #   end
+  #
+  # post /importers/create
+  #----------------------------------------------------------------------------
+  #   def create
+  #     @error = false
+  #     @result = {
+  #         items: [],
+  #         errors: []
+  #     }
+  #
+  #     if params[:importer]
+  #       @importer = Importer.create(import_params)
+  #       if @importer.valid?
+  #         @importer.save
+  #        @result = FatFreeCRM::ImportHandle.process(@importer)
+  #       else
+  #         puts @importer.errors.full_messages
+  #         @result[:errors].push(@importer.errors.full_messages)
+  #         @error = true
+  #       end
+  #     end
+  #     respond_with(@error,@result)
+  #   end
+
+  private
+
+  def importer_params
+    params.require(:importer).permit(:attachment, :entity_type, :entity_id)
+  end
+end

app/models/files/imported_file.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Copyright (c) 2008-2013 Michael Dvorkin and contributors.
+#
+# Fat Free CRM is freely distributable under the terms of MIT license.
+# See MIT-LICENSE file or http://www.opensource.org/licenses/mit-license.php
+#------------------------------------------------------------------------------
+# == Schema Information
+#
+# Table name: imported_files
+#
+#  id              :integer         not null, primary key
+#  filename        :string(64)      default(""), not null
+#  md5sum          :string(32)      default(""), not null
+#
+
+class ImportedFile < ActiveRecord::Base
+  before_validation :generate_md5sum
+
+  validate  :filetype
+
+  validates :filename, presence: true
+  validates :md5sum, presence: true
+  validates :md5sum, uniqueness: { message: "file already imported" }
+
+  def generate_md5sum
+    self.md5sum = Digest::MD5.hexdigest File.open(filename).read unless filename.empty?
+  rescue StandardError
+    ""
+  end
+
+  private
+
+  def filetype
+    valid = begin
+              File.open(filename).type_from_file_command == "application/vnd.ms-excel"
+            rescue StandardError
+              ""
+            end
+    errors.add(:filename, "no such file") if valid == ""
+    errors.add(:filename, "invalid filetype") unless valid
+  end
+
+  ActiveSupport.run_load_hooks(:fat_free_crm_imported_file, self)
+end

app/models/files/importer.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: importers
+#
+#  id                       :integer         not null, primary key
+#  entity_type              :string
+#  entity_id                :integer
+#  attachment_file_size    :integer
+#  attachment_file_name    :string(255)
+#  attachment_content_type :string(255)
+#  status                  :string(255)
+#  created_at              :datetime
+#  updated_at              :datetime
+#
+require 'json'
+class Importer < ActiveRecord::Base
+  attribute :entity_attrs
+
+  has_attached_file :attachment, path: ":rails_root/public/importers/:id/:filename"
+
+  validates_attachment :attachment, presence: true
+
+  validates_attachment_content_type :attachment,
+                                    content_type: %w[text/xml application/xml application/vnd.ms-excel application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/x-ole-storage],
+                                    message: 'Only EXCEL files are allowed.'
+  validates_attachment_file_name :attachment, matches: [/\.xls/, /\.xlsx?$/]
+
+  def messages
+    JSON.parse(messages)
+  end
+
+  def entity_class
+    entity_type.capitalize.constantize
+  end
+
+  ActiveSupport.run_load_hooks(:fat_free_crm_importer, self)
+end

app/views/campaigns/_list_title_bar.html.haml

@@ -0,0 +1,14 @@
+- model_name = controller_name.singularize
+- model_klass = model_name.camelcase.constantize
+
+.title_tools
+  #buttons
+    = view_buttons
+  .create_asset
+    = link_to_inline("create_#{model_name}".to_sym, send("new_#{model_name}_path"), text: t("create_#{model_name}".to_sym))
+  .create_asset
+    = link_to_inline(:new_importer, new_importer_path(:campaign), text: t(:import_campaigns))
+
+.title
+  %span{id: "create_#{model_name}_title"} #{t controller_name.to_sym}
+  = image_tag("loading.gif", size: :thumb, id: "loading", style: "display: none;")