Update readme with more next steps

The cache was not working properly as it intercepted all the emails
except for the first receiver.

.gitignore

@@ -1,5 +1,6 @@
-.env
+.env*
 *.db
 *.sqlite*
 README.html
 .yardoc/
+bin/db_seed.local

Gemfile

@@ -3,30 +3,27 @@
 source "https://rubygems.org"
 
 # Mails proto
-gem "net-smtp", "~> 0.3.3"
 gem "net-imap", "~> 0.3.1"
+gem "net-smtp", "~> 0.3.3"
 
 # Mail template
 gem "erb", "~> 3.0"
 
 # Database
-gem "sqlite3", "~> 1.5"
 gem "sequel", "~> 5.62"
-gem "colorize", "~> 0.8.1"
+gem "sqlite3", "~> 1.5"
 
 # Web
-gem "sinatra", "~> 3.0"
 gem "puma", "~> 6.0"
+gem "sinatra", "~> 3.0"
 gem "slim", "~> 4.1"
 
 # Others
 gem "dotenv", "~> 2.8"
 gem "uuid", "~> 2.3"
-gem "semver", "~> 1.0"
 
 # Debug and stuff
 group :develop do
+  gem "mocha", "~> 2.0"
   gem "pry", "~> 0.14.1"
 end
-
-gem "mocha", "~> 2.0"

Gemfile.lock

@@ -3,7 +3,6 @@ GEM
   specs:
     cgi (0.3.3)
     coderay (1.1.3)
-    colorize (0.8.1)
     dotenv (2.8.1)
     erb (3.0.0)
       cgi (>= 0.3.3)
@@ -30,7 +29,6 @@ GEM
     rack-protection (3.0.3)
       rack
     ruby2_keywords (0.0.5)
-    semver (1.0.1)
     sequel (5.62.0)
     sinatra (3.0.3)
       mustermann (~> 3.0)
@@ -52,7 +50,6 @@ PLATFORMS
   x86_64-linux
 
 DEPENDENCIES
-  colorize (~> 0.8.1)
   dotenv (~> 2.8)
   erb (~> 3.0)
   mocha (~> 2.0)
@@ -60,7 +57,6 @@ DEPENDENCIES
   net-smtp (~> 0.3.3)
   pry (~> 0.14.1)
   puma (~> 6.0)
-  semver (~> 1.0)
   sequel (~> 5.62)
   sinatra (~> 3.0)
   slim (~> 4.1)

README.md

@@ -37,6 +37,30 @@ Copy and fill all the env variables in .env
     cp empty.env .env
     edit .env
 
+Here is the list of all the configuration available (sampled in empty.env):
+
+    SMTP_HOST      information to connect on the smtp
+    SMTP_PORT
+    SMTP_TLS       true or false
+    SMTP_USER
+    SMTP_PASSWORD
+    IMAP_HOST      only imap available
+    IMAP_PORT
+    IMAP_TLS
+    IMAP_USER
+    IMAP_PASSWORD
+    SENDER_HOST    the domain of the sender field, and the mailinglist addresses
+    SENDER         "true" to retrive the true sender, "list" to use the mailing list email
+                   else it is a static email address that will alway be used.
+    FROM           same as SENDER
+    DB_URL         sqlite://db/database.sqlite for instance
+    PORT           for WWW accees, not used yet
+    DEBUG          true/false
+    CPU_SLEEP      slow down the distributor
+    LOG_FILE       log stuff in the specified file
+
+
+
 ### Setup the database
 
     bin/db_migrate
@@ -53,11 +77,25 @@ After deploying it, there are some tools:
 
 - `bin/db_seed` to generate some data
 
+### Additional resources
+
+[How to manage mailinglists](doc/manage_mailinglists.md)
+
 ## Contributing
 
+### Next step
+
+- [ ] Handle multi-modo (currently admin operations send the mail to the first modo only)
+- [ ] Add web interface (for archives)
+- [ ] Auto cut citations and mailinglist signatures
+- [ ] Fetch history via email
+- [ ] Global admin system via email
+- [ ] Harder email security (check output server to verify authenticity)
+- [ ] Disable signature by mailinglist
+
 ### via Gitea
 
-1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
+1. Fork it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
 2. Create your feature branch (`git checkout -b my-new-feature`)
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
@@ -65,9 +103,10 @@ After deploying it, there are some tools:
 
 ### via emails
 
-Checkout git-send-mail tutorial <https://git-send-email.io/>
+Checkout git-send-mail tutorial <https://git-send-email.io/>.
+The mailing list is <mailto:list+mailinglistrb@sceptique.eu>
 
-1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglistrb>)
+1. Clone it (<https://git.sceptique.eu/Sceptique/mailinglist.rb>)
-2. Subscribe to the mailinglist to send your patch <mailto:list.mailinglistrb@sceptique.eu?subject=subscribe> (don't send your patch in this email)
+2. Commit your changes (`git commit -am 'Add some feature'`)
-3. Commit your changes (`git commit -am 'Add some feature'`)
+3. Register the mailinglist by sending a "subscribe" to it
-4. Send your email to <mailto:list.mailinglistrb@sceptique.eu> after you are validated by modo
+4. Send your patch to the mailing list

Rakefile

@@ -7,6 +7,34 @@ Minitest::TestTask.create(:test) do |t|
   t.test_globs = ["test/**/test_*.rb"]
 end
 
+namespace "db" do
+  desc "Migrate the database to the lasted schema"
+  task "migrate" do
+    load "bin/db_migrate"
+  end
+
+  desc "Initialize database with dumby data"
+  task "dumb_seed" do
+    load "bin/db_seed"
+  end
+
+  desc "Initialize database with local data"
+  task "local_seed" do
+    load "bin/db_seed.local"
+  end
+
+  desc "Reset all tables, schema, data"
+  task "reset" do
+    require_relative "lib/app"
+    $db.tables.each { $db.drop_table _1 }
+  end
+
+  namespace "reset" do
+    task "stuff" do
+    end
+  end
+end
+
 task :default => :test
 
 task default: :test

bin/db_migrate

@@ -1,87 +1,7 @@
 #!/usr/bin/env ruby
 $LOAD_PATH << File.join(Dir.pwd, "lib")
 
-require "semver"
-require "colorize"
 require "app"
-require "semver"
 
-def current_version
+sequel_command = "bundle exec sequel -E -m db/migrations #{ENV['DB_URL']}"
-  $db[:meta].first[:version] rescue 0
+exec(sequel_command)
-end
-
-def migrate(version, message = nil, &block)
-  puts "Check migration #{version}".on_blue
-  if current_version < version
-    puts "Migrate #{version}".blue
-    begin
-      yield
-      $db[:meta].update(version: version)
-      puts "Successfuly set version #{version}".green
-      puts message.green if message
-    rescue => err
-      puts err.message.on_red
-      exit 1
-    end
-  else
-    puts "Already migrated #{version}".yellow
-  end
-end
-
-migrate 1, "Initialized database" do
-  # this table only contains one entry
-  $db.create_table :meta do
-    primary_key :id
-    Int :version
-    String :code_version
-    String :code_date
-  end
-  $db[:meta].insert(version: 1)
-end rescue puts "already initialized".yellow
-
-migrate 2, "Initialize mailinglists" do
-  $db.create_table :mailinglists do
-    primary_key :id
-    column :name,                  String, null: false # Name to display
-    column :email,                 String, null: false # Mailing list email
-    column :count_handled,         Integer  # amount of email distributed
-    column :count_distributed,     Integer  # amount of email distributed
-    column :last_email,            DateTime # date of last distribution
-    column :strategy,              String, null: false # "free/validated/closed/..."
-    column :updated_at,            DateTime
-    column :created_at,            DateTime
-
-    index :name, unique: true
-    index :email, unique: true
-  end
-end rescue puts "mailinglists already exists".yellow
-
-migrate 3, "Initialize emails" do
-  $db.create_table :emails do
-    primary_key :id
-    foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
-    column :name,           String, null: false, fixed: true, size: 50
-    column :email,          String, null: false, fixed: true, size: 254
-    column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
-    column :updated_at,     DateTime
-    column :created_at,     DateTime
-
-    index %i[email], unique: false # list personnal subscriptions
-    index %i[email mailinglist_id], unique: true
-    index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
-    index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
-  end
-end rescue puts "emails already exists".yellow
-
-puts "End migration".on_blue
-code_version = `git tag`.split("\n").map{ |str| Semver.new(str) }.sort.last
-code_date = if code_version
-  `git show #{code_version} --pretty="format:%as"`.split("\n").first
-else
-  Time.now
-end
-code_version ||= "v0.0.0"
-$db[:meta].update(code_version: code_version.to_s)
-$db[:meta].update(code_date: code_date.to_s)
-puts "Set code version to #{code_version}".green
-puts "Set code date to #{code_date}".green

db/migrations/0001_create_mailinglists.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :mailinglists do
+      primary_key :id
+      column :name,                  String, null: false # Name to display
+      column :email,                 String, null: false # Mailing list email
+      column :count_handled,         Integer  # amount of email distributed
+      column :count_distributed,     Integer  # amount of email distributed
+      column :last_email,            DateTime # date of last distribution
+      column :strategy,              String, null: false # "free/validated/closed/..."
+      column :updated_at,            DateTime
+      column :created_at,            DateTime
+
+      index :name, unique: true
+      index :email, unique: true
+    end
+  end
+end

db/migrations/0002_create_emails.rb

@@ -0,0 +1,18 @@
+Sequel.migration do
+  change do
+    create_table :emails do
+      primary_key :id
+      foreign_key :mailinglist_id, :mailinglists, null: false #, on_delete: :cascade
+      column :name,           String, null: false, fixed: true, size: 50
+      column :email,          String, null: false, fixed: true, size: 254
+      column :permissions,    Integer # bitfield. 1=read 2=write 4=admin
+      column :updated_at,     DateTime
+      column :created_at,     DateTime
+
+      index %i[email], unique: false # list personnal subscriptions
+      index %i[email mailinglist_id], unique: true
+      index %i[mailinglist_id], unique: false # allow to search entries by mailinglist
+      index %i[mailinglist_id permissions], unique: false # find readers, writers, etc.
+    end
+  end
+end

doc/manage_mailinglists.md

@@ -0,0 +1,43 @@
+# Mailinglist management
+
+## What are strategies?
+
+There are 2 types of strategies: registration and moderation.
+Registration defines how new user can access the list.
+Moderation defines how user can write on the list.
+
+### Registration
+
+- autoregister: writing any email to the mailinglist will add the email of the user to the mailinglist
+- free: any email can register to the list by sending a "subscribe" email to the list
+- validated: like free but moderators needs to confirm the email with a "set-permissions" email. the user details are distributed to moderators first.
+- closed: nobody can register via email
+
+### Moderation
+
+- freewrite: anyone registered can distribute emails to the list
+- restrictedwrite: email needs approval by moderator before distribution. the email details are only distributed to moderators first.
+
+## How to interact with the list by email ?
+
+You need to send an email to the list with the subject being an action.
+Some action can take parameters in the subject or the body.
+
+Actions are:
+- set-permissions
+- validate
+- refuse
+- subscribe
+- unsubscribe
+- help
+- list-users
+- owner
+
+Parameters are key=value, which are separated with "," in the subject or "new line" in the body.
+
+### permissions parameters
+permissions: a permission mask (0=reader, 1=writer, 2=operator, 4=moderator)
+user-email: the email of the user
+
+### validate parameters
+uid: uid of the email

empty.env

@@ -11,6 +11,8 @@ IMAP_USER=
 IMAP_PASSWORD=
 
 SENDER_HOST=machin.fr
+SENDER=true
+FROM=true
 DB_URL="sqlite://dev.db"
 
 PORT=10081

lib/distributor.rb

@@ -39,7 +39,7 @@ class Distributor
       "refuse"          => Actions::RefuseDistribute.new(distributor: self),
     }
   end
-  attr_reader :imap_client, :smtp_client, :handlers if $debug
+  attr_reader :imap_client, :smtp_client, :handlers # if $debug # TODO: more alias for imap/smtp clients?
 
   # Make sure any incoming email is properly directed to the right action.
   def handle_one(mail)
@@ -50,7 +50,7 @@ class Distributor
       attributes = Attributes.parse(subject: subject_attributes, body: mail.body)
       handler = @handlers[subject] || @handlers[:default]
       $logger.info "#{handler.class}#handle on <#{list.email}> for <#{mail.from}>"
-      handler.handle(list:, mail:, attributes:)
+      handler.handle(list:, mail: mail, attributes:)
     else
       $logger.warn "list #{mail.to} do not exist (asked by #{mail.from})"
     end

lib/distributor/#subscribed.txt.erb#

@@ -1 +0,0 @@
-You have subscribed to <%= list.name %> <<%= list.email %>> 

lib/distributor/action.rb

@@ -11,7 +11,7 @@ class Distributor
         @distributor = distributor
       end
 
-      def handle(list:, to:, attributes:)
+      def handle(list:, mail:, attributes:)
         $logger.error "#{self.class} is not implemented yet"
       end
     end

lib/distributor/actions/admin.rb

@@ -91,7 +91,22 @@ class Distributor
       end
     end
 
+    LIST_USERS_TEMPLATE = Actions.template("list_users")
     class ListUsers < Action
+      def handle(list:, mail:, attributes:)
+        modo = Email.first(mailinglist: list, email: mail.from)
+        if !modo&.modo? && !modo&.op?
+          $logger.warn "SECU <#{mail.from}> failed to set-permissions <#{list.email}> modo"
+          return nil
+        end
+
+        body = LIST_USERS_TEMPLATE.result binding
+        @distributor.distribute(
+          Protocols::Mail.build(
+            subject: "ML #{list.name} user list", list:, to: modo.email, body:
+          )
+        )
+      end
     end
 
   end

lib/distributor/actions/user.rb

@@ -82,8 +82,8 @@ class Distributor
         if list.registration?("autoregister")
           if user.nil?
             $logger.info "registering <#{mail.from}> to <#{list.email}>"
-            Email.register!(mailinglist: list, name: mail.from_name || mail.from.split("@").first,
+            user = Email.register!(mailinglist: list, name: mail.from_name || mail.from.split("@").first,
-                            email: mail.from).save
+                                   email: mail.from).save
           end
           # ok let the mail pass the security check
         elsif !user&.writer?
@@ -111,7 +111,8 @@ class Distributor
         # TODO: this is the most lazy code I wrote this week I think
         # if multipart, get the boundary to add a new part in the top
         boundary = mail.header("Content-Type").to_s.match(/boundary="([^"]+)"/)[1] rescue nil
-        body = WAIT_MODO_DISTRIBUTE_TEMPLATE.result binding
+        wait_modo_distribute_template = Actions.template("distribute.wait_modo")
+        body = wait_modo_distribute_template.result binding
         modo_mail = Protocols::Mail.build(
           subject: "ML #{list.name} requires validation for [#{mail.subject}]",
           list:, to: modo.email, body:,

lib/distributor/templates/distribute.wait_modo.txt.erb

@@ -1,5 +1,4 @@
 <% if boundary %>--<%= boundary %>
-Content-Transfer-Encoding: quoted-printable
 Content-Type: text/plain; charset="utf-8"; protected-headers="v1"
 
 <% end %><%= mail.from %> Tried to send an email to <%= list.email %>

lib/distributor/templates/list_users.txt.erb

@@ -0,0 +1,5 @@
+List of the email subscribed to <%= list.name %> <<%= list.email %>>:
+
+<% list.emails.each do |email| %>
+  - <%= email.name %> <<%= email.email %>> (permissions=<%= email.permissions %>, created <%= email.created_at %>)
+<% end %>

lib/models/mailinglist.rb

@@ -1,7 +1,7 @@
 require "uuid"
 
 class Mailinglist < Sequel::Model($db)
-  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "."
+  SUFFIX_SEPARATOR = ENV["MAILINGLIST_SUFFIX_SEPARATOR"] || "+"
   BASE_USER = ENV["MAILINGLIST_BASE_USER"] || "mailinglist"
   HOST = ENV["MAILINGLIST_HOST"]
 
@@ -86,7 +86,7 @@ class Mailinglist < Sequel::Model($db)
   ACTIONS_EMAILS = %i[
     help subscribe unsubscribe owner
     set_permissions list_users
-  ]
+  ].freeze
   def help_email
     "#{email}?subject=help"
   end

lib/protocols/imap.rb

@@ -2,7 +2,7 @@ require "net/imap"
 
 # Protocls::Imap allows to fetch emails.
 #
-# https://www.rfc-editor.org/rfc/rfc3501#page-54
+# https://www.rfc-editor.org/rfc/rfc3501
 #
 # TODO: strengthen the network management to avoid connection loss.
 class Protocols::Imap
@@ -17,7 +17,7 @@ class Protocols::Imap
   attr_reader :imap if $debug
 
   def reset!
-    @imap = Net::IMAP.new(ENV["IMAP_HOST"], ENV["IMAP_PORT"], ssl: true)
+    @imap = Net::IMAP.new(ENV["IMAP_HOST"], port: ENV["IMAP_PORT"], ssl: true)
     @imap.authenticate('PLAIN', ENV["IMAP_USER"], ENV["IMAP_PASSWORD"])
     ALL_INBOXES.each do
       $logger.info "Try to create inbox #{_1}"
@@ -31,10 +31,18 @@ class Protocols::Imap
   # Fetch the next incomming email as a Protocols::Mail.
   #
   # @return [Protocols::Mail]
-  def fetch_next_unseen(inbox:)
+  def fetch_next_unseen(inbox:, max_tries: 2)
-    uid = search_unseen(inbox:).first
+    return $logger.error("fetch_next_unseen reached max_tries_limit") if max_tries == 0
-    return nil if uid.nil?
 
+    begin
+      uid = search_unseen(inbox:).first
+    rescue IOError => e
+      $logger.warn e.message
+      reset!
+      return fetch_next_unseen(inbox:, max_tries: max_tries - 1)
+    end
+
+    return nil if uid.nil?
     fetch(uid:, inbox:)
   end
 

lib/protocols/mail.rb

@@ -11,8 +11,11 @@ class Protocols::Mail
   HEADERS_KEEP = %w[
     Content-Type Content-Transfer-Encoding MIME-Version
   ].freeze
-  HEADERS_KEEP_WITH_ORIGINAL = HEADERS_KEEP + %w[Message-ID User-Agent From To Subject Subject Date In-Reply-To References]
+  HEADERS_KEEP_WITH_ORIGINAL = HEADERS_KEEP + %w[Message-ID User-Agent From To Subject Subject Date In-Reply-To
+                                                 References]
   USER_AGENT = "Ruby/#{RUBY_VERSION} (#{RUBY_PLATFORM}) Mailinglist.rb/1.0".freeze
+  SENDER = ENV["SENDER"]
+  FROM = ENV["FROM"]
 
   # @params imap_mail is any element outputed in the returned array of
   #   Net::IMAP#fetch, and must contains some attributes:
@@ -57,6 +60,11 @@ class Protocols::Mail
     end
   end
 
+  # This method is a kind of hash.
+  def cache_id
+    "#{@message_id} #{@from} #{@to}"
+  end
+
   # Transform an received email into a mailing list email.
   # Tested with evolution and Kmail.
   # Thunderbird bug at the moment which make it weird but it works too.
@@ -65,14 +73,34 @@ class Protocols::Mail
   # @param dest [Email]
   def to_redistribute(list:, dest:)
     new = clone
+    sender =
+      if SENDER == "true"
+        @from
+      elsif SENDER == "list"
+        list.email
+      else
+        SENDER
+      end
+
+    from_email =
+      if FROM == "true"
+        @from
+      elsif FROM == "list"
+        list.email
+      else
+        FROM
+      end
+
+    from = "\"#{@from_name}\" (via #{list.name}) <#{from_email}>"
+
     new.replace_headers!(
       ["User-Agent",       USER_AGENT],
       # require people to do not respond privatly
       ["Reply-to",         "#{list.name} <#{list.email}>"],
       ["Errors-To",        list.email],
       ["To",               dest.email],
-      ["From",             "\"#{@from_name}\" (via #{list.name}) <#{@from}>"],
+      ["From",             from],
-      ["Sender",           @from],
+      ["Sender",           sender ],
       ["Date",             Time.now.strftime(DATE_FORMAT)],
       ["List-Id",          "<#{list.email}>"],
       ["List-Post",        "<mailto:#{list.email}>"],

lib/protocols/smtp.rb

@@ -8,7 +8,7 @@ class Protocols::Smtp
   end
 
   class DistributedCache < Array
-    def initialize(max_size: 100)
+    def initialize(max_size: 4096)
       @max_size = max_size
       super()
     end
@@ -25,7 +25,7 @@ class Protocols::Smtp
 
   # @param mail [Protocols::Mail]
   def distribute(mail)
-    if cache.include?(mail.message_id)
+    if cache.include?(mail.cache_id)
       $logger.warn "Already distributed #{mail.message_id}"
       return
     end
@@ -34,7 +34,7 @@ class Protocols::Smtp
     smtp_raw = mail.to_smtp
     $logger.debug smtp_raw.join("=====")
     send_message_safe(*smtp_raw)
-    cache << mail.message_id
+    cache << mail.cache_id
   rescue StandardError => e
     $logger.warn "FAILED #{mail.from}\t -> #{mail.to}:\t#{mail.subject}"
     $logger.debug e
@@ -61,6 +61,9 @@ class Protocols::Smtp
       $logger.warn e.message
       reset_smtp_client!
       send_message_safe(*raw, max_tries: max_tries - 1)
+    rescue => e
+      $logger.error e.full_message
+      reset_smtp_client!
     end
   end
 end

lib/semver.rb

@@ -1,44 +0,0 @@
-class Semver
-  def initialize(string)
-    @values = string.gsub(/^v(\d.+)/, '\1').split(".").map(&:to_i)
-  end
-
-  def [](index)
-    @values[index] || 0
-  end
-
-  def to_s
-    "v#{@values.join('.')}"
-  end
-
-  def <=>(other)
-    cmp = 0
-    @values.each_with_index do |value, index|
-      cmp = value - other[index]
-      break if cmp != 0
-    end
-    cmp
-  end
-
-  def >(other)
-    (self <=> other) > 0
-  end
-
-  def <(other)
-    (self <=> other) < 0
-  end
-
-  def ==(other)
-    (self <=> other) == 0
-  end
-
-  def !=(other)
-    (self <=> other) != 0
-  end
-end
-
-# pp Utils::Semver.new("1.0") < Utils::Semver.new("1.1")
-# pp Utils::Semver.new("0.1") < Utils::Semver.new("1.1")
-# pp Utils::Semver.new("1.1") < Utils::Semver.new("1.2")
-# pp Utils::Semver.new("1.1") < Utils::Semver.new("2.0")
-# pp Utils::Semver.new("12.0") > Utils::Semver.new("2.2")