searchengines-web/MeiliSearch
searchengines-web/MeiliSearch
1
0
Fork 0
mirror of https://github.com/meilisearch/MeiliSearch synced 2024-11-22 12:54:26 +01:00
Code Issues Releases Wiki Activity

write the compat layer from v2 to v3

Browse Source
This commit is contained in:
Tamo 2022-10-10 14:32:11 +02:00 committed by Clément Renault
parent 6107540ad4
commit 06fadb3004
No known key found for this signature in database
GPG Key ID: 92ADA4E935E71FA4
19 changed files with 1737 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dump/src/error.rs
View File

@ -4,8 +4,10 @@ use thiserror::Error;
pub enum Error {
#[error("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")]
DumpV1Unsupported,
#[error("Bad index name")]
#[error("Bad index name.")]
BadIndexName,
#[error("Malformed task.")]
MalformedTask,
#[error(transparent)]
Io(#[from] std::io::Error),

1
dump/src/reader/compat/mod.rs
View File

@ -13,6 +13,7 @@ use super::{
v6::{self, V6IndexReader, V6Reader},
};
pub mod v2_to_v3;
pub mod v3_to_v4;
pub mod v4_to_v5;
pub mod v5_to_v6;

31
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap Normal file
View File

@ -0,0 +1,31 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: spells.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{},
),
sortable_attributes: NotSet,
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"exactness",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{},
),
distinct_attribute: Reset,
_kind: PhantomData<dump::reader::v3::settings::Checked>,
},
)

533
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-12.snap Normal file
View File

@ -0,0 +1,533 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: documents
---
[
{
"index": "acid-arrow",
"name": "Acid Arrow",
"desc": [
"A shimmering green arrow streaks toward a target within range and bursts in a spray of acid. Make a ranged spell attack against the target. On a hit, the target takes 4d4 acid damage immediately and 2d4 acid damage at the end of its next turn. On a miss, the arrow splashes the target with acid for half as much of the initial damage and no damage at the end of its next turn."
],
"higher_level": [
"When you cast this spell using a spell slot of 3rd level or higher, the damage (both initial and later) increases by 1d4 for each slot level above 2nd."
],
"range": "90 feet",
"components": [
"V",
"S",
"M"
],
"material": "Powdered rhubarb leaf and an adder's stomach.",
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"attack_type": "ranged",
"damage": {
"damage_type": {
"index": "acid",
"name": "Acid",
"url": "/api/damage-types/acid"
},
"damage_at_slot_level": {
"2": "4d4",
"3": "5d4",
"4": "6d4",
"5": "7d4",
"6": "8d4",
"7": "9d4",
"8": "10d4",
"9": "11d4"
}
},
"school": {
"index": "evocation",
"name": "Evocation",
"url": "/api/magic-schools/evocation"
},
"classes": [
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
},
{
"index": "land",
"name": "Land",
"url": "/api/subclasses/land"
}
],
"url": "/api/spells/acid-arrow"
},
{
"index": "acid-splash",
"name": "Acid Splash",
"desc": [
"You hurl a bubble of acid. Choose one creature within range, or choose two creatures within range that are within 5 feet of each other. A target must succeed on a dexterity saving throw or take 1d6 acid damage.",
"This spell's damage increases by 1d6 when you reach 5th level (2d6), 11th level (3d6), and 17th level (4d6)."
],
"range": "60 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 action",
"level": 0,
"damage": {
"damage_type": {
"index": "acid",
"name": "Acid",
"url": "/api/damage-types/acid"
},
"damage_at_character_level": {
"1": "1d6",
"5": "2d6",
"11": "3d6",
"17": "4d6"
}
},
"school": {
"index": "conjuration",
"name": "Conjuration",
"url": "/api/magic-schools/conjuration"
},
"classes": [
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/acid-splash",
"dc": {
"dc_type": {
"index": "dex",
"name": "DEX",
"url": "/api/ability-scores/dex"
},
"dc_success": "none"
}
},
{
"index": "aid",
"name": "Aid",
"desc": [
"Your spell bolsters your allies with toughness and resolve. Choose up to three creatures within range. Each target's hit point maximum and current hit points increase by 5 for the duration."
],
"higher_level": [
"When you cast this spell using a spell slot of 3rd level or higher, a target's hit points increase by an additional 5 for each slot level above 2nd."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A tiny strip of white cloth.",
"ritual": false,
"duration": "8 hours",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "abjuration",
"name": "Abjuration",
"url": "/api/magic-schools/abjuration"
},
"classes": [
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "paladin",
"name": "Paladin",
"url": "/api/classes/paladin"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/aid",
"heal_at_slot_level": {
"2": "5",
"3": "10",
"4": "15",
"5": "20",
"6": "25",
"7": "30",
"8": "35",
"9": "40"
}
},
{
"index": "alarm",
"name": "Alarm",
"desc": [
"You set an alarm against unwanted intrusion. Choose a door, a window, or an area within range that is no larger than a 20-foot cube. Until the spell ends, an alarm alerts you whenever a Tiny or larger creature touches or enters the warded area. When you cast the spell, you can designate creatures that won't set off the alarm. You also choose whether the alarm is mental or audible.",
"A mental alarm alerts you with a ping in your mind if you are within 1 mile of the warded area. This ping awakens you if you are sleeping.",
"An audible alarm produces the sound of a hand bell for 10 seconds within 60 feet."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A tiny bell and a piece of fine silver wire.",
"ritual": true,
"duration": "8 hours",
"concentration": false,
"casting_time": "1 minute",
"level": 1,
"school": {
"index": "abjuration",
"name": "Abjuration",
"url": "/api/magic-schools/abjuration"
},
"classes": [
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/alarm",
"area_of_effect": {
"type": "cube",
"size": 20
}
},
{
"index": "alter-self",
"name": "Alter Self",
"desc": [
"You assume a different form. When you cast the spell, choose one of the following options, the effects of which last for the duration of the spell. While the spell lasts, you can end one option as an action to gain the benefits of a different one.",
"***Aquatic Adaptation.*** You adapt your body to an aquatic environment, sprouting gills and growing webbing between your fingers. You can breathe underwater and gain a swimming speed equal to your walking speed.",
"***Change Appearance.*** You transform your appearance. You decide what you look like, including your height, weight, facial features, sound of your voice, hair length, coloration, and distinguishing characteristics, if any. You can make yourself appear as a member of another race, though none of your statistics change. You also can't appear as a creature of a different size than you, and your basic shape stays the same; if you're bipedal, you can't use this spell to become quadrupedal, for instance. At any time for the duration of the spell, you can use your action to change your appearance in this way again.",
"***Natural Weapons.*** You grow claws, fangs, spines, horns, or a different natural weapon of your choice. Your unarmed strikes deal 1d6 bludgeoning, piercing, or slashing damage, as appropriate to the natural weapon you chose, and you are proficient with your unarmed strikes. Finally, the natural weapon is magic and you have a +1 bonus to the attack and damage rolls you make using it."
],
"range": "Self",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 1 hour",
"concentration": true,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/alter-self"
},
{
"index": "animal-friendship",
"name": "Animal Friendship",
"desc": [
"This spell lets you convince a beast that you mean it no harm. Choose a beast that you can see within range. It must see and hear you. If the beast's Intelligence is 4 or higher, the spell fails. Otherwise, the beast must succeed on a wisdom saving throw or be charmed by you for the spell's duration. If you or one of your companions harms the target, the spells ends."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A morsel of food.",
"ritual": false,
"duration": "24 hours",
"concentration": false,
"casting_time": "1 action",
"level": 1,
"school": {
"index": "enchantment",
"name": "Enchantment",
"url": "/api/magic-schools/enchantment"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
},
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
}
],
"subclasses": [],
"url": "/api/spells/animal-friendship",
"dc": {
"dc_type": {
"index": "wis",
"name": "WIS",
"url": "/api/ability-scores/wis"
},
"dc_success": "none"
}
},
{
"index": "animal-messenger",
"name": "Animal Messenger",
"desc": [
"By means of this spell, you use an animal to deliver a message. Choose a Tiny beast you can see within range, such as a squirrel, a blue jay, or a bat. You specify a location, which you must have visited, and a recipient who matches a general description, such as \"a man or woman dressed in the uniform of the town guard\" or \"a red-haired dwarf wearing a pointed hat.\" You also speak a message of up to twenty-five words. The target beast travels for the duration of the spell toward the specified location, covering about 50 miles per 24 hours for a flying messenger, or 25 miles for other animals.",
"When the messenger arrives, it delivers your message to the creature that you described, replicating the sound of your voice. The messenger speaks only to a creature matching the description you gave. If the messenger doesn't reach its destination before the spell ends, the message is lost, and the beast makes its way back to where you cast this spell."
],
"higher_level": [
"If you cast this spell using a spell slot of 3nd level or higher, the duration of the spell increases by 48 hours for each slot level above 2nd."
],
"range": "30 feet",
"components": [
"V",
"S",
"M"
],
"material": "A morsel of food.",
"ritual": true,
"duration": "24 hours",
"concentration": false,
"casting_time": "1 action",
"level": 2,
"school": {
"index": "enchantment",
"name": "Enchantment",
"url": "/api/magic-schools/enchantment"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
},
{
"index": "ranger",
"name": "Ranger",
"url": "/api/classes/ranger"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/animal-messenger"
},
{
"index": "animal-shapes",
"name": "Animal Shapes",
"desc": [
"Your magic turns others into beasts. Choose any number of willing creatures that you can see within range. You transform each target into the form of a Large or smaller beast with a challenge rating of 4 or lower. On subsequent turns, you can use your action to transform affected creatures into new forms.",
"The transformation lasts for the duration for each target, or until the target drops to 0 hit points or dies. You can choose a different form for each target. A target's game statistics are replaced by the statistics of the chosen beast, though the target retains its alignment and Intelligence, Wisdom, and Charisma scores. The target assumes the hit points of its new form, and when it reverts to its normal form, it returns to the number of hit points it had before it transformed. If it reverts as a result of dropping to 0 hit points, any excess damage carries over to its normal form. As long as the excess damage doesn't reduce the creature's normal form to 0 hit points, it isn't knocked unconscious. The creature is limited in the actions it can perform by the nature of its new form, and it can't speak or cast spells.",
"The target's gear melds into the new form. The target can't activate, wield, or otherwise benefit from any of its equipment."
],
"range": "30 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 24 hours",
"concentration": true,
"casting_time": "1 action",
"level": 8,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "druid",
"name": "Druid",
"url": "/api/classes/druid"
}
],
"subclasses": [],
"url": "/api/spells/animal-shapes"
},
{
"index": "animate-dead",
"name": "Animate Dead",
"desc": [
"This spell creates an undead servant. Choose a pile of bones or a corpse of a Medium or Small humanoid within range. Your spell imbues the target with a foul mimicry of life, raising it as an undead creature. The target becomes a skeleton if you chose bones or a zombie if you chose a corpse (the DM has the creature's game statistics).",
"On each of your turns, you can use a bonus action to mentally command any creature you made with this spell if the creature is within 60 feet of you (if you control multiple creatures, you can command any or all of them at the same time, issuing the same command to each one). You decide what action the creature will take and where it will move during its next turn, or you can issue a general command, such as to guard a particular chamber or corridor. If you issue no commands, the creature only defends itself against hostile creatures. Once given an order, the creature continues to follow it until its task is complete.",
"The creature is under your control for 24 hours, after which it stops obeying any command you've given it. To maintain control of the creature for another 24 hours, you must cast this spell on the creature again before the current 24-hour period ends. This use of the spell reasserts your control over up to four creatures you have animated with this spell, rather than animating a new one."
],
"higher_level": [
"When you cast this spell using a spell slot of 4th level or higher, you animate or reassert control over two additional undead creatures for each slot level above 3rd. Each of the creatures must come from a different corpse or pile of bones."
],
"range": "10 feet",
"components": [
"V",
"S",
"M"
],
"material": "A drop of blood, a piece of flesh, and a pinch of bone dust.",
"ritual": false,
"duration": "Instantaneous",
"concentration": false,
"casting_time": "1 minute",
"level": 3,
"school": {
"index": "necromancy",
"name": "Necromancy",
"url": "/api/magic-schools/necromancy"
},
"classes": [
{
"index": "cleric",
"name": "Cleric",
"url": "/api/classes/cleric"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [
{
"index": "lore",
"name": "Lore",
"url": "/api/subclasses/lore"
}
],
"url": "/api/spells/animate-dead"
},
{
"index": "animate-objects",
"name": "Animate Objects",
"desc": [
"Objects come to life at your command. Choose up to ten nonmagical objects within range that are not being worn or carried. Medium targets count as two objects, Large targets count as four objects, Huge targets count as eight objects. You can't animate any object larger than Huge. Each target animates and becomes a creature under your control until the spell ends or until reduced to 0 hit points.",
"As a bonus action, you can mentally command any creature you made with this spell if the creature is within 500 feet of you (if you control multiple creatures, you can command any or all of them at the same time, issuing the same command to each one). You decide what action the creature will take and where it will move during its next turn, or you can issue a general command, such as to guard a particular chamber or corridor. If you issue no commands, the creature only defends itself against hostile creatures. Once given an order, the creature continues to follow it until its task is complete.",
"##### Animated Object Statistics",
"| Size | HP | AC | Attack | Str | Dex |",
"|---|---|---|---|---|---|",
"| Tiny | 20 | 18 | +8 to hit, 1d4 + 4 damage | 4 | 18 |",
"| Small | 25 | 16 | +6 to hit, 1d8 + 2 damage | 6 | 14 |",
"| Medium | 40 | 13 | +5 to hit, 2d6 + 1 damage | 10 | 12 |",
"| Large | 50 | 10 | +6 to hit, 2d10 + 2 damage | 14 | 10 |",
"| Huge | 80 | 10 | +8 to hit, 2d12 + 4 damage | 18 | 6 |",
"An animated object is a construct with AC, hit points, attacks, Strength, and Dexterity determined by its size. Its Constitution is 10 and its Intelligence and Wisdom are 3, and its Charisma is 1. Its speed is 30 feet; if the object lacks legs or other appendages it can use for locomotion, it instead has a flying speed of 30 feet and can hover. If the object is securely attached to a surface or a larger object, such as a chain bolted to a wall, its speed is 0. It has blindsight with a radius of 30 feet and is blind beyond that distance. When the animated object drops to 0 hit points, it reverts to its original object form, and any remaining damage carries over to its original object form.",
"If you command an object to attack, it can make a single melee attack against a creature within 5 feet of it. It makes a slam attack with an attack bonus and bludgeoning damage determined by its size. The DM might rule that a specific object inflicts slashing or piercing damage based on its form."
],
"higher_level": [
"If you cast this spell using a spell slot of 6th level or higher, you can animate two additional objects for each slot level above 5th."
],
"range": "120 feet",
"components": [
"V",
"S"
],
"ritual": false,
"duration": "Up to 1 minute",
"concentration": true,
"casting_time": "1 action",
"level": 5,
"school": {
"index": "transmutation",
"name": "Transmutation",
"url": "/api/magic-schools/transmutation"
},
"classes": [
{
"index": "bard",
"name": "Bard",
"url": "/api/classes/bard"
},
{
"index": "sorcerer",
"name": "Sorcerer",
"url": "/api/classes/sorcerer"
},
{
"index": "wizard",
"name": "Wizard",
"url": "/api/classes/wizard"
}
],
"subclasses": [],
"url": "/api/spells/animate-objects"
}
]

200
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-2.snap Normal file
View File

@ -0,0 +1,200 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: tasks
---
[
{
"uuid": "5867e9f1-1ebb-4145-b0ec-b61b29da43e9",
"update": {
"status": "enqueued",
"updateId": 0,
"meta": {
"DocumentAddition": {
"primary_key": null,
"method": "ReplaceDocuments",
"content_uuid": "1d0832b1-02f7-4c56-849e-d150d266ab46"
}
},
"enqueuedAt": "2022-10-09T20:27:59.828915609Z"
}
},
{
"uuid": "5661371a-21ab-4363-b2e5-5ca66126e5e0",
"update": {
"status": "processed",
"success": "Other",
"processedAt": "2022-10-09T20:27:22.688964637Z",
"updateId": 0,
"meta": {
"Settings": {
"synonyms": {
"android": [
"phone",
"smartphone"
],
"iphone": [
"phone",
"smartphone"
],
"phone": [
"smartphone",
"iphone",
"android"
]
}
}
},
"enqueuedAt": "2022-10-09T20:27:22.597296237Z",
"startedProcessingAt": "2022-10-09T20:27:22.610066114Z"
}
},
{
"uuid": "5661371a-21ab-4363-b2e5-5ca66126e5e0",
"update": {
"status": "failed",
"updateId": 1,
"meta": {
"DocumentAddition": {
"primary_key": null,
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:23.305963122Z",
"startedProcessingAt": "2022-10-09T20:27:23.312497053Z",
"msg": "missing primary key",
"code": "UnretrievableErrorCode",
"failedAt": "2022-10-09T20:27:23.31297828Z"
}
},
{
"uuid": "5661371a-21ab-4363-b2e5-5ca66126e5e0",
"update": {
"status": "processed",
"success": {
"DocumentsAddition": {
"nb_documents": 10
}
},
"processedAt": "2022-10-09T20:27:23.951017769Z",
"updateId": 2,
"meta": {
"DocumentAddition": {
"primary_key": "sku",
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:23.91528854Z",
"startedProcessingAt": "2022-10-09T20:27:23.921493715Z"
}
},
{
"uuid": "bb33e237-be17-453c-83a2-4fef67d03220",
"update": {
"status": "processed",
"success": {
"DocumentsAddition": {
"nb_documents": 10
}
},
"processedAt": "2022-10-09T20:27:22.197788495Z",
"updateId": 0,
"meta": {
"DocumentAddition": {
"primary_key": null,
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:22.075264451Z",
"startedProcessingAt": "2022-10-09T20:27:22.085751162Z"
}
},
{
"uuid": "bb33e237-be17-453c-83a2-4fef67d03220",
"update": {
"status": "processed",
"success": "Other",
"processedAt": "2022-10-09T20:27:22.411761344Z",
"updateId": 1,
"meta": {
"Settings": {
"rankingRules": [
"words",
"typo",
"proximity",
"attribute",
"exactness",
"asc(release_date)"
]
}
},
"enqueuedAt": "2022-10-09T20:27:22.380218549Z",
"startedProcessingAt": "2022-10-09T20:27:22.393023806Z"
}
},
{
"uuid": "bb33e237-be17-453c-83a2-4fef67d03220",
"update": {
"status": "processed",
"success": {
"DocumentsAddition": {
"nb_documents": 100
}
},
"processedAt": "2022-10-09T20:28:01.93111053Z",
"updateId": 2,
"meta": {
"DocumentAddition": {
"primary_key": null,
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:59.817923645Z",
"startedProcessingAt": "2022-10-09T20:27:59.829038211Z"
}
},
{
"uuid": "f20c9936-a26e-4960-8a1f-3bdb390608f2",
"update": {
"status": "failed",
"updateId": 0,
"meta": {
"DocumentAddition": {
"primary_key": null,
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:24.157663206Z",
"startedProcessingAt": "2022-10-09T20:27:24.162839906Z",
"msg": "missing primary key",
"code": "UnretrievableErrorCode",
"failedAt": "2022-10-09T20:27:24.242683494Z"
}
},
{
"uuid": "f20c9936-a26e-4960-8a1f-3bdb390608f2",
"update": {
"status": "processed",
"success": {
"DocumentsAddition": {
"nb_documents": 10
}
},
"processedAt": "2022-10-09T20:27:24.312809641Z",
"updateId": 1,
"meta": {
"DocumentAddition": {
"primary_key": "index",
"method": "ReplaceDocuments",
"content_uuid": "00112233-4455-6677-8899-aabbccddeeff"
}
},
"enqueuedAt": "2022-10-09T20:27:24.283289037Z",
"startedProcessingAt": "2022-10-09T20:27:24.285985108Z"
}
}
]

45
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-4.snap Normal file
View File

@ -0,0 +1,45 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: products.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{},
),
sortable_attributes: NotSet,
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"exactness",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{
"android": [
"phone",
"smartphone",
],
"iphone": [
"phone",
"smartphone",
],
"phone": [
"android",
"iphone",
"smartphone",
],
},
),
distinct_attribute: Reset,
_kind: PhantomData<dump::reader::v3::settings::Checked>,
},
)

308
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap Normal file
View File

@ -0,0 +1,308 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: documents
---
[
{
"sku": 127687,
"name": "Duracell - AA Batteries (8-Pack)",
"type": "HardGood",
"price": 7.49,
"upc": "041333825014",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; AA size; DURALOCK Power Preserve technology; 8-pack",
"manufacturer": "Duracell",
"model": "MN1500B8Z",
"url": "http://www.bestbuy.com/site/duracell-aa-batteries-8-pack/127687.p?id=1051384045676&skuId=127687&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1276/127687_sa.jpg"
},
{
"sku": 150115,
"name": "Energizer - MAX Batteries AA (4-Pack)",
"type": "HardGood",
"price": 4.99,
"upc": "039800011329",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "4-pack AA alkaline batteries; battery tester included",
"manufacturer": "Energizer",
"model": "E91BP-4",
"url": "http://www.bestbuy.com/site/energizer-max-batteries-aa-4-pack/150115.p?id=1051384046217&skuId=150115&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1501/150115_sa.jpg"
},
{
"sku": 185230,
"name": "Duracell - C Batteries (4-Pack)",
"type": "HardGood",
"price": 8.99,
"upc": "041333440019",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; C size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN1400R4Z",
"url": "http://www.bestbuy.com/site/duracell-c-batteries-4-pack/185230.p?id=1051384046486&skuId=185230&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1852/185230_sa.jpg"
},
{
"sku": 185267,
"name": "Duracell - D Batteries (4-Pack)",
"type": "HardGood",
"price": 9.99,
"upc": "041333430010",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.99,
"description": "Compatible with select electronic devices; D size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN1300R4Z",
"url": "http://www.bestbuy.com/site/duracell-d-batteries-4-pack/185267.p?id=1051384046551&skuId=185267&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/1852/185267_sa.jpg"
},
{
"sku": 312290,
"name": "Duracell - 9V Batteries (2-Pack)",
"type": "HardGood",
"price": 7.99,
"upc": "041333216010",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; alkaline chemistry; 9V size; DURALOCK Power Preserve technology; 2-pack",
"manufacturer": "Duracell",
"model": "MN1604B2Z",
"url": "http://www.bestbuy.com/site/duracell-9v-batteries-2-pack/312290.p?id=1051384050321&skuId=312290&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3122/312290_sa.jpg"
},
{
"sku": 324884,
"name": "Directed Electronics - Viper Audio Glass Break Sensor",
"type": "HardGood",
"price": 39.99,
"upc": "093207005060",
"category": [
{
"id": "pcmcat113100050015",
"name": "Carfi Instore Only"
}
],
"shipping": 0,
"description": "From our expanded online assortment; compatible with Directed Electronics alarm systems; microphone and microprocessor detect and analyze intrusions; detects quiet glass breaks",
"manufacturer": "Directed Electronics",
"model": "506T",
"url": "http://www.bestbuy.com/site/directed-electronics-viper-audio-glass-break-sensor/324884.p?id=1112808077651&skuId=324884&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3248/324884_rc.jpg"
},
{
"sku": 333179,
"name": "Energizer - N Cell E90 Batteries (2-Pack)",
"type": "HardGood",
"price": 5.99,
"upc": "039800013200",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208006",
"name": "Specialty Batteries"
}
],
"shipping": 5.49,
"description": "Alkaline batteries; 1.5V",
"manufacturer": "Energizer",
"model": "E90BP-2",
"url": "http://www.bestbuy.com/site/energizer-n-cell-e90-batteries-2-pack/333179.p?id=1185268509951&skuId=333179&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3331/333179_sa.jpg"
},
{
"sku": 346575,
"name": "Metra - Radio Installation Dash Kit for Most 1989-2000 Ford, Lincoln & Mercury Vehicles - Black",
"type": "HardGood",
"price": 16.99,
"upc": "086429002757",
"category": [
{
"id": "abcat0300000",
"name": "Car Electronics & GPS"
},
{
"id": "pcmcat165900050023",
"name": "Car Installation Parts & Accessories"
},
{
"id": "pcmcat331600050007",
"name": "Car Audio Installation Parts"
},
{
"id": "pcmcat165900050031",
"name": "Deck Installation Parts"
},
{
"id": "pcmcat165900050033",
"name": "Dash Installation Kits"
}
],
"shipping": 0,
"description": "From our expanded online assortment; compatible with most 1989-2000 Ford, Lincoln and Mercury vehicles; snap-in TurboKit offers fast installation; spacer/trim ring; rear support bracket",
"manufacturer": "Metra",
"model": "99-5512",
"url": "http://www.bestbuy.com/site/metra-radio-installation-dash-kit-for-most-1989-2000-ford-lincoln-mercury-vehicles-black/346575.p?id=1218118704590&skuId=346575&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/3465/346575_rc.jpg"
},
{
"sku": 43900,
"name": "Duracell - AAA Batteries (4-Pack)",
"type": "HardGood",
"price": 5.49,
"upc": "041333424019",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Compatible with select electronic devices; AAA size; DURALOCK Power Preserve technology; 4-pack",
"manufacturer": "Duracell",
"model": "MN2400B4Z",
"url": "http://www.bestbuy.com/site/duracell-aaa-batteries-4-pack/43900.p?id=1051384074145&skuId=43900&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/4390/43900_sa.jpg"
},
{
"sku": 48530,
"name": "Duracell - AA 1.5V CopperTop Batteries (4-Pack)",
"type": "HardGood",
"price": 5.49,
"upc": "041333415017",
"category": [
{
"id": "pcmcat312300050015",
"name": "Connected Home & Housewares"
},
{
"id": "pcmcat248700050021",
"name": "Housewares"
},
{
"id": "pcmcat303600050001",
"name": "Household Batteries"
},
{
"id": "abcat0208002",
"name": "Alkaline Batteries"
}
],
"shipping": 5.49,
"description": "Long-lasting energy; DURALOCK Power Preserve technology; for toys, clocks, radios, games, remotes, PDAs and more",
"manufacturer": "Duracell",
"model": "MN1500B4Z",
"url": "http://www.bestbuy.com/site/duracell-aa-1-5v-coppertop-batteries-4-pack/48530.p?id=1099385268988&skuId=48530&cmp=RMXCC",
"image": "http://img.bbystatic.com/BestBuy_US/images/products/4853/48530_sa.jpg"
}
]

31
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap Normal file
View File

@ -0,0 +1,31 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: movies2.settings()
---
Ok(
Settings {
displayed_attributes: Reset,
searchable_attributes: Reset,
filterable_attributes: Set(
{},
),
sortable_attributes: NotSet,
ranking_rules: Set(
[
"words",
"typo",
"proximity",
"attribute",
"exactness",
],
),
stop_words: Set(
{},
),
synonyms: Set(
{},
),
distinct_attribute: Reset,
_kind: PhantomData<dump::reader::v3::settings::Checked>,
},
)

5
dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-9.snap Normal file
View File

@ -0,0 +1,5 @@
---
source: dump/src/reader/compat/v2_to_v3.rs
expression: documents
---
[]

465
dump/src/reader/compat/v2_to_v3.rs Normal file
View File

@ -0,0 +1,465 @@
use std::convert::TryInto;
use std::str::FromStr;
use time::OffsetDateTime;
use uuid::Uuid;
use crate::reader::{v2, v3, DumpReader, IndexReader};
use crate::Result;
use super::v3_to_v4::CompatV3ToV4;
pub struct CompatV2ToV3 {
pub from: v2::V2Reader,
}
impl CompatV2ToV3 {
pub fn new(v2: v2::V2Reader) -> CompatV2ToV3 {
CompatV2ToV3 { from: v2 }
}
pub fn index_uuid(&self) -> Vec<v3::meta::IndexUuid> {
self.index_uuid()
.into_iter()
.map(|index| v3::meta::IndexUuid {
uid: index.uid,
uuid: index.uuid,
})
.collect()
}
pub fn to_v4(self) -> CompatV3ToV4 {
CompatV3ToV4::Compat(self)
}
pub fn version(&self) -> crate::Version {
self.from.version()
}
pub fn date(&self) -> Option<time::OffsetDateTime> {
self.from.date()
}
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
Ok(None)
}
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV2ToV3>> + '_> {
Ok(self.from.indexes()?.map(|index_reader| -> Result<_> {
let compat = CompatIndexV2ToV3::new(index_reader?);
Ok(compat)
}))
}
pub fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(v3::Task, Option<v3::UpdateFile>)>> + '_> {
let indexes = self.from.index_uuid.clone();
Box::new(
self.from
.tasks()
.map(move |task| {
task.map(|(task, content_file)| {
let task = v3::Task {
uuid: task.uuid,
update: task.update.into(),
};
Some((task, content_file))
})
})
.filter_map(|res| res.transpose()),
)
}
}
pub struct CompatIndexV2ToV3 {
from: v2::V2IndexReader,
}
impl CompatIndexV2ToV3 {
pub fn new(v2: v2::V2IndexReader) -> CompatIndexV2ToV3 {
CompatIndexV2ToV3 { from: v2 }
}
pub fn metadata(&self) -> &crate::IndexMetadata {
self.from.metadata()
}
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<v3::Document>> + '_>> {
self.from
.documents()
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v3::Document>> + '_>)
}
pub fn settings(&mut self) -> Result<v3::Settings<v3::Checked>> {
Ok(v3::Settings::<v3::Unchecked>::from(self.from.settings()?).check())
}
}
impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
fn from(update: v2::updates::UpdateStatus) -> Self {
match update {
v2::updates::UpdateStatus::Processing(processing) => {
match (processing.from.meta.clone(), processing.from.content).try_into() {
Ok(meta) => v3::updates::UpdateStatus::Processing(v3::updates::Processing {
from: v3::updates::Enqueued {
update_id: processing.from.update_id,
meta,
enqueued_at: processing.from.enqueued_at,
},
started_processing_at: processing.started_processing_at,
}),
Err(e) => {
log::warn!("Error with task {}: {}", processing.from.update_id, e);
log::warn!("Task will be marked as `Failed`.");
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
from: v3::updates::Processing {
from: v3::updates::Enqueued {
update_id: processing.from.update_id,
meta: update_from_unchecked_update_meta(processing.from.meta),
enqueued_at: processing.from.enqueued_at,
},
started_processing_at: processing.started_processing_at,
},
msg: e.to_string(),
code: v3::Code::MalformedDump,
failed_at: OffsetDateTime::now_utc(),
})
}
}
}
v2::updates::UpdateStatus::Enqueued(enqueued) => {
match (enqueued.meta.clone(), enqueued.content).try_into() {
Ok(meta) => v3::updates::UpdateStatus::Enqueued(v3::updates::Enqueued {
update_id: enqueued.update_id,
meta,
enqueued_at: enqueued.enqueued_at,
}),
Err(e) => {
log::warn!("Error with task {}: {}", enqueued.update_id, e);
log::warn!("Task will be marked as `Failed`.");
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
from: v3::updates::Processing {
from: v3::updates::Enqueued {
update_id: enqueued.update_id,
meta: update_from_unchecked_update_meta(enqueued.meta),
enqueued_at: enqueued.enqueued_at,
},
started_processing_at: OffsetDateTime::now_utc(),
},
msg: e.to_string(),
code: v3::Code::MalformedDump,
failed_at: OffsetDateTime::now_utc(),
})
}
}
}
v2::updates::UpdateStatus::Processed(processed) => {
v3::updates::UpdateStatus::Processed(v3::updates::Processed {
success: processed.success.into(),
processed_at: processed.processed_at,
from: v3::updates::Processing {
from: v3::updates::Enqueued {
update_id: processed.from.from.update_id,
// since we're never going to read the content_file again it's ok to generate a fake one.
meta: update_from_unchecked_update_meta(processed.from.from.meta),
enqueued_at: processed.from.from.enqueued_at,
},
started_processing_at: processed.from.started_processing_at,
},
})
}
v2::updates::UpdateStatus::Aborted(aborted) => {
v3::updates::UpdateStatus::Aborted(v3::updates::Aborted {
from: v3::updates::Enqueued {
update_id: aborted.from.update_id,
// since we're never going to read the content_file again it's ok to generate a fake one.
meta: update_from_unchecked_update_meta(aborted.from.meta),
enqueued_at: aborted.from.enqueued_at,
},
aborted_at: aborted.aborted_at,
})
}
v2::updates::UpdateStatus::Failed(failed) => {
v3::updates::UpdateStatus::Failed(v3::updates::Failed {
from: v3::updates::Processing {
from: v3::updates::Enqueued {
update_id: failed.from.from.update_id,
// since we're never going to read the content_file again it's ok to generate a fake one.
meta: update_from_unchecked_update_meta(failed.from.from.meta),
enqueued_at: failed.from.from.enqueued_at,
},
started_processing_at: failed.from.started_processing_at,
},
msg: failed.error.message,
code: failed.error.error_code.into(),
failed_at: failed.failed_at,
})
}
}
}
}
impl TryFrom<(v2::updates::UpdateMeta, Option<Uuid>)> for v3::updates::Update {
type Error = crate::Error;
fn try_from((update, uuid): (v2::updates::UpdateMeta, Option<Uuid>)) -> Result<Self> {
Ok(match update {
v2::updates::UpdateMeta::DocumentsAddition {
method,
format: _,
primary_key,
} if uuid.is_some() => v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
content_uuid: uuid.unwrap(),
},
v2::updates::UpdateMeta::DocumentsAddition { .. } => {
return Err(crate::Error::MalformedTask)
}
v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
v2::updates::UpdateMeta::DeleteDocuments { ids } => {
v3::updates::Update::DeleteDocuments(ids)
}
v2::updates::UpdateMeta::Settings(settings) => {
v3::updates::Update::Settings(settings.into())
}
})
}
}
pub fn update_from_unchecked_update_meta(update: v2::updates::UpdateMeta) -> v3::updates::Update {
match update {
v2::updates::UpdateMeta::DocumentsAddition {
method,
format,
primary_key,
} => v3::updates::Update::DocumentAddition {
primary_key,
method: match method {
v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
v3::updates::IndexDocumentsMethod::ReplaceDocuments
}
v2::updates::IndexDocumentsMethod::UpdateDocuments => {
v3::updates::IndexDocumentsMethod::UpdateDocuments
}
},
// we use this special uuid so we can recognize it if one day there is a bug related to this field.
content_uuid: Uuid::from_str("00112233-4455-6677-8899-aabbccddeeff").unwrap(),
},
v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
v2::updates::UpdateMeta::DeleteDocuments { ids } => {
v3::updates::Update::DeleteDocuments(ids)
}
v2::updates::UpdateMeta::Settings(settings) => {
v3::updates::Update::Settings(settings.into())
}
}
}
impl From<v2::updates::UpdateResult> for v3::updates::UpdateResult {
fn from(result: v2::updates::UpdateResult) -> Self {
match result {
v2::updates::UpdateResult::DocumentsAddition(addition) => {
v3::updates::UpdateResult::DocumentsAddition(v3::updates::DocumentAdditionResult {
nb_documents: addition.nb_documents,
})
}
v2::updates::UpdateResult::DocumentDeletion { deleted } => {
v3::updates::UpdateResult::DocumentDeletion { deleted }
}
v2::updates::UpdateResult::Other => v3::updates::UpdateResult::Other,
}
}
}
impl From<String> for v3::Code {
fn from(code: String) -> Self {
match code.as_ref() {
"CreateIndex" => v3::Code::CreateIndex,
"IndexAlreadyExists" => v3::Code::IndexAlreadyExists,
"IndexNotFound" => v3::Code::IndexNotFound,
"InvalidIndexUid" => v3::Code::InvalidIndexUid,
"InvalidState" => v3::Code::InvalidState,
"MissingPrimaryKey" => v3::Code::MissingPrimaryKey,
"PrimaryKeyAlreadyPresent" => v3::Code::PrimaryKeyAlreadyPresent,
"MaxFieldsLimitExceeded" => v3::Code::MaxFieldsLimitExceeded,
"MissingDocumentId" => v3::Code::MissingDocumentId,
"InvalidDocumentId" => v3::Code::InvalidDocumentId,
"Filter" => v3::Code::Filter,
"Sort" => v3::Code::Sort,
"BadParameter" => v3::Code::BadParameter,
"BadRequest" => v3::Code::BadRequest,
"DatabaseSizeLimitReached" => v3::Code::DatabaseSizeLimitReached,
"DocumentNotFound" => v3::Code::DocumentNotFound,
"Internal" => v3::Code::Internal,
"InvalidGeoField" => v3::Code::InvalidGeoField,
"InvalidRankingRule" => v3::Code::InvalidRankingRule,
"InvalidStore" => v3::Code::InvalidStore,
"InvalidToken" => v3::Code::InvalidToken,
"MissingAuthorizationHeader" => v3::Code::MissingAuthorizationHeader,
"NoSpaceLeftOnDevice" => v3::Code::NoSpaceLeftOnDevice,
"DumpNotFound" => v3::Code::DumpNotFound,
"TaskNotFound" => v3::Code::TaskNotFound,
"PayloadTooLarge" => v3::Code::PayloadTooLarge,
"RetrieveDocument" => v3::Code::RetrieveDocument,
"SearchDocuments" => v3::Code::SearchDocuments,
"UnsupportedMediaType" => v3::Code::UnsupportedMediaType,
"DumpAlreadyInProgress" => v3::Code::DumpAlreadyInProgress,
"DumpProcessFailed" => v3::Code::DumpProcessFailed,
"InvalidContentType" => v3::Code::InvalidContentType,
"MissingContentType" => v3::Code::MissingContentType,
"MalformedPayload" => v3::Code::MalformedPayload,
"MissingPayload" => v3::Code::MissingPayload,
other => {
log::warn!("Unknown error code {}", other);
v3::Code::UnretrievableErrorCode
}
}
}
}
fn option_to_setting<T>(opt: Option<Option<T>>) -> v3::Setting<T> {
match opt {
Some(Some(t)) => v3::Setting::Set(t),
None => v3::Setting::NotSet,
Some(None) => v3::Setting::Reset,
}
}
impl<T> From<v2::Settings<T>> for v3::Settings<v3::Unchecked> {
fn from(settings: v2::Settings<T>) -> Self {
v3::Settings {
displayed_attributes: option_to_setting(settings.displayed_attributes),
searchable_attributes: option_to_setting(settings.searchable_attributes),
filterable_attributes: option_to_setting(settings.filterable_attributes)
.map(|f| f.into_iter().collect()),
sortable_attributes: v3::Setting::NotSet,
ranking_rules: option_to_setting(settings.ranking_rules),
stop_words: option_to_setting(settings.stop_words),
synonyms: option_to_setting(settings.synonyms),
distinct_attribute: option_to_setting(settings.distinct_attribute),
_kind: std::marker::PhantomData,
}
}
}
#[cfg(test)]
pub(crate) mod test {
use std::{fs::File, io::BufReader};
use flate2::bufread::GzDecoder;
use tempfile::TempDir;
use super::*;
#[test]
fn compat_v2_v3() {
let dump = File::open("tests/assets/v2.dump").unwrap();
let dir = TempDir::new().unwrap();
let mut dump = BufReader::new(dump);
let gz = GzDecoder::new(&mut dump);
let mut archive = tar::Archive::new(gz);
archive.unpack(dir.path()).unwrap();
let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
// top level infos
insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
// tasks
let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
insta::assert_json_snapshot!(tasks);
assert_eq!(update_files.len(), 9);
assert!(update_files[0].is_some()); // the enqueued document addition
assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
// indexes
let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
// the index are not ordered in any way by default
indexes.sort_by_key(|index| index.metadata().uid.to_string());
let mut products = indexes.pop().unwrap();
let mut movies2 = indexes.pop().unwrap();
let movies = indexes.pop().unwrap();
let mut spells = indexes.pop().unwrap();
assert!(indexes.is_empty());
// products
insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "products",
"primaryKey": "sku",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(products.settings());
let documents = products
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 10);
insta::assert_json_snapshot!(documents);
// movies
insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "movies",
"primaryKey": "id",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
// movies2
insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "movies_2",
"primaryKey": null,
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(movies2.settings());
let documents = movies2
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 0);
insta::assert_debug_snapshot!(documents);
// spells
insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
{
"uid": "dnd_spells",
"primaryKey": "index",
"createdAt": "[now]",
"updatedAt": "[now]"
}
"###);
insta::assert_debug_snapshot!(spells.settings());
let documents = spells
.documents()
.unwrap()
.collect::<Result<Vec<_>>>()
.unwrap();
assert_eq!(documents.len(), 10);
insta::assert_json_snapshot!(documents);
}
}

95
dump/src/reader/compat/v3_to_v4.rs
View File

@ -1,15 +1,17 @@
use crate::reader::{v3, v4, DumpReader, IndexReader};
use crate::Result;
use super::v2_to_v3::{CompatIndexV2ToV3, CompatV2ToV3};
use super::v4_to_v5::CompatV4ToV5;
pub struct CompatV3ToV4 {
pub from: v3::V3Reader,
pub enum CompatV3ToV4 {
V3(v3::V3Reader),
Compat(CompatV2ToV3),
}
impl CompatV3ToV4 {
pub fn new(v3: v3::V3Reader) -> CompatV3ToV4 {
CompatV3ToV4 { from: v3 }
CompatV3ToV4::V3(v3)
}
pub fn to_v5(self) -> CompatV4ToV5 {
@ -17,11 +19,17 @@ impl CompatV3ToV4 {
}
pub fn version(&self) -> crate::Version {
self.from.version()
match self {
CompatV3ToV4::V3(v3) => v3.version(),
CompatV3ToV4::Compat(compat) => compat.version(),
}
}
pub fn date(&self) -> Option<time::OffsetDateTime> {
self.from.date()
match self {
CompatV3ToV4::V3(v3) => v3.date(),
CompatV3ToV4::Compat(compat) => compat.date(),
}
}
pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
@ -29,20 +37,36 @@ impl CompatV3ToV4 {
}
pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV3ToV4>> + '_> {
Ok(self.from.indexes()?.map(|index_reader| -> Result<_> {
let compat = CompatIndexV3ToV4::new(index_reader?);
Ok(compat)
}))
Ok(match self {
CompatV3ToV4::V3(v3) => Box::new(
v3.indexes()?
.map(|index| index.map(CompatIndexV3ToV4::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>,
CompatV3ToV4::Compat(compat) => Box::new(
compat
.indexes()?
.map(|index| index.map(CompatIndexV3ToV4::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>,
})
}
pub fn tasks(
&mut self,
) -> Box<dyn Iterator<Item = Result<(v4::Task, Option<v4::UpdateFile>)>> + '_> {
let indexes = self.from.index_uuid.clone();
let indexes = match self {
CompatV3ToV4::V3(v3) => v3.index_uuid(),
CompatV3ToV4::Compat(compat) => compat.index_uuid(),
};
let tasks = match self {
CompatV3ToV4::V3(v3) => v3.tasks(),
CompatV3ToV4::Compat(compat) => compat.tasks(),
};
Box::new(
self.from
.tasks()
tasks
.map(move |task| {
task.map(|(task, content_file)| {
let index_uid = indexes
@ -188,27 +212,56 @@ impl CompatV3ToV4 {
}
}
pub struct CompatIndexV3ToV4 {
from: v3::V3IndexReader,
pub enum CompatIndexV3ToV4 {
V3(v3::V3IndexReader),
Compat(CompatIndexV2ToV3),
}
impl From<v3::V3IndexReader> for CompatIndexV3ToV4 {
fn from(index_reader: v3::V3IndexReader) -> Self {
Self::V3(index_reader)
}
}
impl From<CompatIndexV2ToV3> for CompatIndexV3ToV4 {
fn from(index_reader: CompatIndexV2ToV3) -> Self {
Self::Compat(index_reader)
}
}
impl CompatIndexV3ToV4 {
pub fn new(v3: v3::V3IndexReader) -> CompatIndexV3ToV4 {
CompatIndexV3ToV4 { from: v3 }
CompatIndexV3ToV4::V3(v3)
}
pub fn metadata(&self) -> &crate::IndexMetadata {
self.from.metadata()
match self {
CompatIndexV3ToV4::V3(v3) => v3.metadata(),
CompatIndexV3ToV4::Compat(compat) => compat.metadata(),
}
}
pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<v4::Document>> + '_>> {
self.from
.documents()
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>)
match self {
CompatIndexV3ToV4::V3(v3) => v3
.documents()
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
CompatIndexV3ToV4::Compat(compat) => compat
.documents()
.map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
}
}
pub fn settings(&mut self) -> Result<v4::Settings<v4::Checked>> {
Ok(v4::Settings::<v4::Unchecked>::from(self.from.settings()?).check())
Ok(match self {
CompatIndexV3ToV4::V3(v3) => {
v4::Settings::<v4::Unchecked>::from(v3.settings()?).check()
}
CompatIndexV3ToV4::Compat(compat) => {
v4::Settings::<v4::Unchecked>::from(compat.settings()?).check()
}
})
}
}
@ -260,6 +313,8 @@ impl From<v3::Code> for v4::Code {
v3::Code::MissingContentType => v4::Code::MissingContentType,
v3::Code::MalformedPayload => v4::Code::MalformedPayload,
v3::Code::MissingPayload => v4::Code::MissingPayload,
v3::Code::UnretrievableErrorCode => v4::Code::UnretrievableErrorCode,
v3::Code::MalformedDump => v4::Code::MalformedDump,
}
}
}

5
dump/src/reader/compat/v4_to_v5.rs
View File

@ -40,7 +40,7 @@ impl CompatV4ToV5 {
}
pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>> {
let indexes = match self {
Ok(match self {
CompatV4ToV5::V4(v4) => Box::new(
v4.indexes()?
.map(|index| index.map(CompatIndexV4ToV5::from)),
@ -53,8 +53,7 @@ impl CompatV4ToV5 {
.map(|index| index.map(CompatIndexV4ToV5::from)),
)
as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>,
};
Ok(indexes)
})
}
pub fn tasks(

10
dump/src/reader/v2/errors.rs
View File

@ -6,9 +6,9 @@ use serde::Deserialize;
#[serde(rename_all = "camelCase")]
pub struct ResponseError {
#[serde(skip)]
code: StatusCode,
message: String,
error_code: String,
error_type: String,
error_link: String,
pub code: StatusCode,
pub message: String,
pub error_code: String,
pub error_type: String,
pub error_link: String,
}

8
dump/src/reader/v2/mod.rs
View File

@ -41,7 +41,7 @@ use crate::{IndexMetadata, Result, Version};
use self::meta::{DumpMeta, IndexUuid};
use super::IndexReader;
use super::{compat::v2_to_v3::CompatV2ToV3, IndexReader};
pub type Document = serde_json::Map<String, serde_json::Value>;
pub type Settings<T> = settings::Settings<T>;
@ -99,11 +99,13 @@ impl V2Reader {
})
}
/*
pub fn to_v3(self) -> CompatV2ToV3 {
CompatV2ToV3::new(self)
}
*/
pub fn index_uuid(&self) -> Vec<IndexUuid> {
self.index_uuid.clone()
}
pub fn version(&self) -> Version {
Version::V2

6
dump/src/reader/v2/updates.rs
View File

@ -7,8 +7,8 @@ use super::{ResponseError, Settings, Unchecked};
#[derive(Deserialize)]
#[cfg_attr(test, derive(serde::Serialize))]
pub struct UpdateEntry {
uuid: Uuid,
update: UpdateStatus,
pub uuid: Uuid,
pub update: UpdateStatus,
}
impl UpdateEntry {
@ -153,7 +153,7 @@ impl Processing {
#[serde(rename_all = "camelCase")]
pub struct Aborted {
#[serde(flatten)]
from: Enqueued,
pub from: Enqueued,
#[serde(with = "time::serde::rfc3339")]
pub aborted_at: OffsetDateTime,
}

7
dump/src/reader/v3/errors.rs
View File

@ -91,6 +91,9 @@ pub enum Code {
MissingContentType,
MalformedPayload,
MissingPayload,
MalformedDump,
UnretrievableErrorCode,
}
impl Code {
@ -181,6 +184,10 @@ impl Code {
ErrCode::invalid("invalid_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
}
MissingPayload => ErrCode::invalid("missing_payload", StatusCode::BAD_REQUEST),
UnretrievableErrorCode => {
ErrCode::invalid("unretrievable_error_code", StatusCode::BAD_REQUEST)
}
MalformedDump => ErrCode::invalid("malformed_dump", StatusCode::BAD_REQUEST),
}
}

6
dump/src/reader/v3/mod.rs
View File

@ -78,7 +78,7 @@ pub struct V3Reader {
dump: TempDir,
metadata: Metadata,
tasks: BufReader<File>,
pub index_uuid: Vec<IndexUuid>,
index_uuid: Vec<IndexUuid>,
}
impl V3Reader {
@ -100,6 +100,10 @@ impl V3Reader {
})
}
pub fn index_uuid(&self) -> Vec<IndexUuid> {
self.index_uuid.clone()
}
pub fn to_v4(self) -> CompatV3ToV4 {
CompatV3ToV4::new(self)
}

11
dump/src/reader/v3/settings.rs
View File

@ -179,6 +179,17 @@ impl<T> Default for Setting<T> {
}
impl<T> Setting<T> {
pub fn map<U, F>(self, f: F) -> Setting<U>
where
F: FnOnce(T) -> U,
{
match self {
Setting::Set(t) => Setting::Set(f(t)),
Setting::Reset => Setting::Reset,
Setting::NotSet => Setting::NotSet,
}
}
pub fn set(self) -> Option<T> {
match self {
Self::Set(value) => Some(value),

2
dump/src/reader/v4/errors.rs
View File

@ -155,6 +155,7 @@ pub enum Code {
InvalidApiKeyDescription,
UnretrievableErrorCode,
MalformedDump,
}
impl Code {
@ -267,6 +268,7 @@ impl Code {
UnretrievableErrorCode => {
ErrCode::invalid("unretrievable_error_code", StatusCode::BAD_REQUEST)
}
MalformedDump => ErrCode::invalid("malformed_dump", StatusCode::BAD_REQUEST),
}
}