Skip to content

Latest commit

 

History

History
707 lines (641 loc) · 15.7 KB

upgrade-instructions.md

File metadata and controls

707 lines (641 loc) · 15.7 KB
Raw

Upgrade instructions

This document covers every changes an Ops needs to be aware of when running Twake Mail backend mail server.

The following procedures are to take as it, and Linagora, nor its contributors, can not be responsible for any damages generated by following the below procedures.

Before performing these operations, you should ensure to have the skills to conduct the operations, and you should read other software documentation. Do not follow this guide blindly!

Unreleased

Note: this section is in progress. It will be updated during all the development process until the release.

0.11.0

Change default minimum ngram settings for contact indices

Date: 01/10/2024

Issue: linagora#1204

Optional to adapt as the effection is minor.

Minimum ngram (minimum characters to autocomplete) now defaults to 2 (previously was 3).

To adapt the change, we need to migrate contact indices to a new version with the new indices settings.

Migrate Domain contact index

  • Step 1: create a v2 index
PUT /domain_contact_v2
{
  "settings": {
    "index": {
      "max_ngram_diff": "27",
      "number_of_shards": "5",
      "number_of_replicas": "1",
      "analysis": {
        "filter": {
          "ngram_filter": {
            "type": "ngram",
            "min_gram": "2",
            "max_gram": "29"
          },
          "edge_ngram_filter": {
            "type": "edge_ngram",
            "min_gram": "2",
            "max_gram": "29"
          },
          "preserved_ascii_folding_filter": {
            "type": "asciifolding",
            "preserve_original": "true"
          }
        },
        "analyzer": {
          "email_ngram_filter_analyzer": {
            "filter": [
              "ngram_filter",
              "lowercase"
            ],
            "type": "custom",
            "tokenizer": "uax_url_email"
          },
          "rebuilt_keyword": {
            "filter": [
              "lowercase"
            ],
            "type": "custom",
            "tokenizer": "keyword"
          },
          "name_edge_ngram_filter_analyzer": {
            "filter": [
              "edge_ngram_filter",
              "lowercase",
              "preserved_ascii_folding_filter"
            ],
            "type": "custom",
            "tokenizer": "standard"
          }
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "contactId": {
        "type": "keyword"
      },
      "domain": {
        "type": "keyword"
      },
      "email": {
        "type": "text",
        "analyzer": "email_ngram_filter_analyzer",
        "search_analyzer": "rebuilt_keyword"
      },
      "firstname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      },
      "surname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      }
    }
  }
}

Notes: We may need to change number_of_shards and number_of_replicas values if needed (have a look at opensearch.properties).

  • Step 2: Expose the new index under the write alias
POST /_aliases
{
  "actions": [
    {
      "remove": {
        "index": "domain_contact",
        "alias": "domain_contact_write_alias"
      }
    },
    {
      "add": {
        "index": "domain_contact_v2",
        "alias": "domain_contact_write_alias",
        "is_write_index": true
      }
    }
  ]
}
  • Step 3: Reindex v2 from v1 index
POST /_reindex?slices=auto
{
  "source": {
    "index": "domain_contact"
  },
  "dest": {
    "index": "domain_contact_v2"
  }
}
  • Step 4: Expose the new index under the read alias
POST /_aliases
{
  "actions": [
    {
      "remove": {
        "index": "domain_contact",
        "alias": "domain_contact_read_alias"
      }
    },
    {
      "add": {
        "index": "domain_contact_v2",
        "alias": "domain_contact_read_alias"
      }
    }
  ]
}
  • Step 5: Delete v1 index
DELETE /domain_contact

Migrate User contact index

  • Step 1: create a v2 index
PUT /user_contact_v2
{
  "settings": {
    "index": {
      "max_ngram_diff": "27",
      "number_of_shards": "5",
      "analysis": {
        "filter": {
          "ngram_filter": {
            "type": "ngram",
            "min_gram": "2",
            "max_gram": "29"
          },
          "edge_ngram_filter": {
            "type": "edge_ngram",
            "min_gram": "2",
            "max_gram": "29"
          },
          "preserved_ascii_folding_filter": {
            "type": "asciifolding",
            "preserve_original": "true"
          }
        },
        "analyzer": {
          "email_ngram_filter_analyzer": {
            "filter": [
              "ngram_filter",
              "lowercase"
            ],
            "type": "custom",
            "tokenizer": "uax_url_email"
          },
          "rebuilt_keyword": {
            "filter": [
              "lowercase"
            ],
            "type": "custom",
            "tokenizer": "keyword"
          },
          "name_edge_ngram_filter_analyzer": {
            "filter": [
              "edge_ngram_filter",
              "lowercase",
              "preserved_ascii_folding_filter"
            ],
            "type": "custom",
            "tokenizer": "standard"
          }
        }
      },
      "number_of_replicas": "1"
    }
  },
  "mappings": {
    "properties": {
      "accountId": {
        "type": "keyword"
      },
      "contactId": {
        "type": "keyword"
      },
      "email": {
        "type": "text",
        "analyzer": "email_ngram_filter_analyzer",
        "search_analyzer": "rebuilt_keyword"
      },
      "firstname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      },
      "surname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      }
    }
  }
}

Notes: We may need to change number_of_shards and number_of_replicas values if needed (have a look at opensearch.properties).

  • Step 2: Expose the new index under the write alias
POST /_aliases
{
  "actions": [
    {
      "remove": {
        "index": "user_contact",
        "alias": "user_contact_write_alias"
      }
    },
    {
      "add": {
        "index": "user_contact_v2",
        "alias": "user_contact_write_alias",
        "is_write_index": true
      }
    }
  ]
}
  • Step 3: Reindex v2 from v1 index
POST /_reindex?slices=auto
{
  "source": {
    "index": "user_contact"
  },
  "dest": {
    "index": "user_contact_v2"
  }
}
  • Step 4: Expose the new index under the read alias
POST /_aliases
{
  "actions": [
    {
      "remove": {
        "index": "user_contact",
        "alias": "user_contact_read_alias"
      }
    },
    {
      "add": {
        "index": "user_contact_v2",
        "alias": "user_contact_read_alias"
      }
    }
  ]
}
  • Step 5: Delete v1 index
DELETE /user_contact

0.6.5-rc2 version

Change list:

Autocomplete should support normalizing language special characters

Date 14/12/2022

Ticket: linagora#534

Concerned product: Distributed Twake Mail, Distributed ES6 Twake Mail

We changed an analyzer setting for user and domain autocomplete indexes, therefore we need to reindex these two indexes to make the new analyzer applies for existing documents.

We need to perform reindexing with aliases for zero downtime as we normally did.

New indexes settings + mapping (pay attention to change some change-me values - could get those values by getting the old indexes' settings):

Distributed Twake Mail (OpenSearch)

  • new user contact index:
{
	"settings": {
		"number_of_shards": change-me,
		"number_of_replicas": change-me,
		"index.write.wait_for_active_shards": change-me,
		"index": {
			"max_ngram_diff": change-me
		},
		"analysis": {
			"analyzer": {
				"email_ngram_filter_analyzer": {
					"tokenizer": "uax_url_email",
					"filter": ["ngram_filter", "lowercase"]
				},
				"name_edge_ngram_filter_analyzer": {
					"tokenizer": "standard",
					"filter": ["edge_ngram_filter", "lowercase", "preserved_ascii_folding_filter"]
				},
				"rebuilt_keyword": {
					"tokenizer": "keyword",
					"filter": ["lowercase"]
				}
			},
			"filter": {
				"ngram_filter": {
					"type": "ngram",
					"min_gram": change-me,
					"max_gram": change-me
				},
				"edge_ngram_filter": {
					"type": "edge_ngram",
					"min_gram": change-me,
					"max_gram": change-me
				},
				"preserved_ascii_folding_filter": {
					"type": "asciifolding",
					"preserve_original": true
				}
			}
		}
	},
	"mappings": {
		"properties": {
			"accountId": {
				"type": "keyword"
			},
			"contactId": {
				"type": "keyword"
			},
			"email": {
				"type": "text",
				"analyzer": "email_ngram_filter_analyzer",
				"search_analyzer": "rebuilt_keyword"
			},
			"firstname": {
				"type": "text",
				"analyzer": "name_edge_ngram_filter_analyzer",
				"search_analyzer": "standard"
			},
			"surname": {
				"type": "text",
				"analyzer": "name_edge_ngram_filter_analyzer",
				"search_analyzer": "standard"
			}
		}
	}
}
  • new domain contact index:
{
	"settings": {
		"number_of_shards": change-me,
		"number_of_replicas": change-me,
		"index.write.wait_for_active_shards": change-me,
		"index": {
			"max_ngram_diff": change-me
		},
		"analysis": {
			"analyzer": {
				"email_ngram_filter_analyzer": {
					"tokenizer": "uax_url_email",
					"filter": ["ngram_filter", "lowercase"]
				},
				"name_edge_ngram_filter_analyzer": {
					"tokenizer": "standard",
					"filter": ["edge_ngram_filter", "lowercase", "preserved_ascii_folding_filter"]
				},
				"rebuilt_keyword": {
					"tokenizer": "keyword",
					"filter": ["lowercase"]
				}
			},
			"filter": {
				"ngram_filter": {
					"type": "ngram",
					"min_gram": change-me,
					"max_gram": change-me
				},
				"edge_ngram_filter": {
					"type": "edge_ngram",
					"min_gram": change-me,
					"max_gram": change-me
				},
				"preserved_ascii_folding_filter": {
					"type": "asciifolding",
					"preserve_original": true
				}
			}
		}
	},
	"mappings": {
		"properties": {
			"domain": {
				"type": "keyword"
			},
			"contactId": {
				"type": "keyword"
			},
			"email": {
				"type": "text",
				"analyzer": "email_ngram_filter_analyzer",
				"search_analyzer": "rebuilt_keyword"
			},
			"firstname": {
				"type": "text",
				"analyzer": "name_edge_ngram_filter_analyzer",
				"search_analyzer": "standard"
			},
			"surname": {
				"type": "text",
				"analyzer": "name_edge_ngram_filter_analyzer",
				"search_analyzer": "standard"
			}
		}
	}
}

Distributed ES6 Twake Mail (Elasticsearch v6)

  • new user contact index:
{
  "settings": {
    "number_of_shards": change-me,
    "number_of_replicas": change-me,
    "index.write.wait_for_active_shards": change-me,
    "analysis": {
      "analyzer": {
        "email_ngram_filter_analyzer": {
          "tokenizer": "uax_url_email",
          "filter": ["ngram_filter", "lowercase"]
        },
        "name_edge_ngram_filter_analyzer": {
          "tokenizer": "standard",
          "filter": ["edge_ngram_filter", "lowercase", "preserved_ascii_folding_filter"]
        },
        "rebuilt_keyword": {
          "tokenizer": "keyword",
          "filter": ["lowercase"]
        }
      },
      "filter": {
        "ngram_filter": {
          "type": "ngram",
          "min_gram": change-me,
          "max_gram": change-me
        },
        "edge_ngram_filter": {
          "type": "edge_ngram",
          "min_gram": change-me,
          "max_gram": change-me
        },
        "preserved_ascii_folding_filter": {
          "type": "asciifolding",
          "preserve_original": true
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "accountId": {
        "type": "keyword"
      },
      "contactId": {
        "type": "keyword"
      },
      "email": {
        "type": "text",
        "analyzer": "email_ngram_filter_analyzer",
        "search_analyzer": "rebuilt_keyword"
      },
      "firstname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      },
      "surname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      }
    }
  }
}
  • new domain contact index:
{
  "settings": {
    "number_of_shards": change-me,
    "number_of_replicas": change-me,
    "index.write.wait_for_active_shards": change-me,
    "analysis": {
      "analyzer": {
        "email_ngram_filter_analyzer": {
          "tokenizer": "uax_url_email",
          "filter": ["ngram_filter", "lowercase"]
        },
        "name_edge_ngram_filter_analyzer": {
          "tokenizer": "standard",
          "filter": ["edge_ngram_filter", "lowercase", "preserved_ascii_folding_filter"]
        },
        "rebuilt_keyword": {
          "tokenizer": "keyword",
          "filter": ["lowercase"]
        }
      },
      "filter": {
        "ngram_filter": {
          "type": "ngram",
          "min_gram": change-me,
          "max_gram": change-me
        },
        "edge_ngram_filter": {
          "type": "edge_ngram",
          "min_gram": change-me,
          "max_gram": change-me
        },
        "preserved_ascii_folding_filter": {
          "type": "asciifolding",
          "preserve_original": true
        }
      }
    }
  },
  "mappings": {
    "properties": {
      "domain": {
        "type": "keyword"
      },
      "contactId": {
        "type": "keyword"
      },
      "email": {
        "type": "text",
        "analyzer": "email_ngram_filter_analyzer",
        "search_analyzer": "rebuilt_keyword"
      },
      "firstname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      },
      "surname": {
        "type": "text",
        "analyzer": "name_edge_ngram_filter_analyzer",
        "search_analyzer": "standard"
      }
    }
  }
}

0.6.3 version

Change list:

Improve indexing of email address domains

Date 04/10/2022

JIRA: https://issues.apache.org/jira/browse/JAMES-3827

Concerned product: Distributed Twake Mail, Distributed ES6 Twake Mail

Distributed Twake Mail

If you did not finish reindexing data as part of OpenSearch Migration, you can skip this instruction and just reindex data.

Otherwise, please add these fields to the mailbox mapping manually:

curl -X PUT \
  http://OSip:OSport/{$mailboxIndexName}/_mapping \
  -H 'Content-Type: application/json' \
  -d '{
  "properties": {
    "from": {
      "properties": {
        "domain": {
          "type": "text",
          "analyzer": "simple",
          "search_analyzer": "keyword"
        }
      }
    },
    "to": {
      "properties": {
        "domain": {
          "type": "text",
          "analyzer": "simple",
          "search_analyzer": "keyword"
        }
      }
    },
    "cc": {
      "properties": {
        "domain": {
          "type": "text",
          "analyzer": "simple",
          "search_analyzer": "keyword"
        }
      }
    },
    "bcc": {
      "properties": {
        "domain": {
          "type": "text",
          "analyzer": "simple",
          "search_analyzer": "keyword"
        }
      }
    }
  }
}'