#!/usr/bin/perl
#
# This file is part of Koha.
#
# Koha is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# Koha is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Koha; if not, see <https://www.gnu.org/licenses>.

use Modern::Perl;
use Encode;

use Test::More tests => 9;
use Test::NoWarnings;
use Test::Warn;
use Test::Exception;

use t::lib::Mocks;
use t::lib::TestBuilder;

use Test::MockModule;

use JSON::PP ();
use MARC::Record;
use Try::Tiny;
use List::Util qw( any );

use C4::AuthoritiesMarc qw( AddAuthority );
use C4::Biblio;

use Koha::SearchEngine::Elasticsearch;
use Koha::SearchEngine::Elasticsearch::Search;

my $schema = Koha::Database->new->schema;
$schema->storage->txn_begin;

subtest '_read_configuration() tests' => sub {

    plan tests => 16;

    my $configuration;
    t::lib::Mocks::mock_config( 'elasticsearch', undef );

    # 'elasticsearch' missing in configuration
    throws_ok {
        $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    }
    'Koha::Exceptions::Config::MissingEntry',
        'Configuration problem, exception thrown';
    is(
        $@->message,
        "Missing <elasticsearch> entry in koha-conf.xml",
        'Exception message is correct'
    );

    # 'elasticsearch' present but no 'server' entry
    t::lib::Mocks::mock_config( 'elasticsearch', {} );
    throws_ok {
        $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    }
    'Koha::Exceptions::Config::MissingEntry',
        'Configuration problem, exception thrown';
    is(
        $@->message,
        "Missing <elasticsearch>/<server> entry in koha-conf.xml",
        'Exception message is correct'
    );

    # 'elasticsearch' and 'server' entries present, but no 'index_name'
    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server' } );
    throws_ok {
        $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    }
    'Koha::Exceptions::Config::MissingEntry',
        'Configuration problem, exception thrown';
    is(
        $@->message,
        "Missing <elasticsearch>/<index_name> entry in koha-conf.xml",
        'Exception message is correct'
    );

    # Correct configuration, only one server
    t::lib::Mocks::mock_config( 'elasticsearch', { server => 'a_server', index_name => 'index' } );

    $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    is( $configuration->{index_name}, 'index', 'Index configuration parsed correctly' );
    is_deeply( $configuration->{nodes}, ['a_server'], 'Server configuration parsed correctly' );

    # Correct configuration, two servers
    my @servers = ( 'a_server', 'another_server' );
    t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index' } );

    $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    is( $configuration->{index_name}, 'index',  'Index configuration parsed correctly' );
    is( $configuration->{cxn_pool},   'Static', 'cxn_pool configuration set correctly to Static if not specified' );
    is_deeply( $configuration->{nodes}, \@servers, 'Server configuration parsed correctly' );

    t::lib::Mocks::mock_config( 'elasticsearch', { server => \@servers, index_name => 'index', cxn_pool => 'Sniff' } );

    $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    is( $configuration->{cxn_pool}, 'Sniff', 'cxn_pool configuration parsed correctly' );
    isnt( defined $configuration->{trace_to}, 'trace_to is not defined if not set' );

    my $params = Koha::SearchEngine::Elasticsearch::get_elasticsearch_params;
    is_deeply(
        $configuration->{nodes}, \@servers,
        'get_elasticsearch_params is just a wrapper for _read_configuration'
    );

    t::lib::Mocks::mock_config(
        'elasticsearch',
        {
            server          => \@servers, index_name => 'index', cxn_pool => 'Sniff', trace_to => 'Stderr',
            request_timeout => 42
        }
    );

    $configuration = Koha::SearchEngine::Elasticsearch::_read_configuration;
    is( $configuration->{trace_to},        'Stderr', 'trace_to configuration parsed correctly' );
    is( $configuration->{request_timeout}, '42',     'additional configuration (request_timeout) parsed correctly' );
};

subtest 'get_elasticsearch_settings() tests' => sub {

    plan tests => 1;

    my $settings;

    # test reading index settings
    my $es = Koha::SearchEngine::Elasticsearch->new( { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
    $settings = $es->get_elasticsearch_settings();
    is(
        $settings->{index}{analysis}{analyzer}{analyzer_phrase}{tokenizer}, 'keyword',
        'Index settings parsed correctly'
    );
};

subtest 'get_elasticsearch_mappings() tests' => sub {

    plan tests => 5;

    my $mappings;

    my @mappings = (
        {
            name        => 'cn-sort',
            type        => 'callnumber',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '001',
        },
        {
            name        => 'isbn',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            filter      => 'punctuation',
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '020a',
        },
    );
    my $search_engine_module = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $search_engine_module->mock(
        '_foreach_mapping',
        sub {
            my ( $self, $sub ) = @_;

            foreach my $map (@mappings) {
                $sub->(
                    $map->{name},
                    $map->{type},
                    $map->{facet},
                    $map->{suggestible},
                    $map->{sort},
                    $map->{searchable},
                    $map->{filter},
                    $map->{marc_type},
                    $map->{marc_field}
                );
            }
        }
    );

    my $search_engine_elasticsearch = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );
    $mappings = $search_engine_elasticsearch->get_elasticsearch_mappings();

    is(
        $mappings->{properties}{"cn-sort__sort"}{index}, 'false',
        'Field mappings parsed correctly for sort for callnumber type'
    );
    is(
        $mappings->{properties}{"cn-sort__sort"}{numeric}, 'false',
        'Field mappings parsed correctly for sort for callnumber type'
    );
    is( $mappings->{properties}{isbn__sort}{index}, 'false', 'Field mappings parsed correctly' );
    is(
        $mappings->{date_detection}, JSON::PP::false,
        'date_detection disabled at mapping root'
    );
    is(
        $mappings->{numeric_detection}, JSON::PP::false,
        'numeric_detection disabled at mapping root'
    );

};

subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () tests' => sub {

    plan tests => 71;

    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );

    my @mappings = (
        {
            name        => 'control_number',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => undef,
            marc_type   => 'marc21',
            marc_field  => '001',
        },
        {
            name        => 'isbn',
            type        => 'isbn',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '020a',
        },
        {
            name        => 'author',
            type        => 'string',
            facet       => 1,
            suggestible => 1,
            searchable  => 1,
            sort        => undef,
            marc_type   => 'marc21',
            marc_field  => '100a',
        },
        {
            name        => 'author',
            type        => 'string',
            facet       => 1,
            suggestible => 1,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '110a',
        },
        {
            name        => 'title',
            type        => 'string',
            facet       => 0,
            suggestible => 1,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '245(ab)ab',
        },
        {
            name        => 'title-no-punctuation',
            type        => 'string',
            facet       => 0,
            suggestible => 1,
            searchable  => 1,
            sort        => undef,
            filter      => 'punctuation',
            marc_type   => 'marc21',
            marc_field  => '245(ab)ab',
        },
        {
            name        => 'unimarc_title',
            type        => 'string',
            facet       => 0,
            suggestible => 1,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'unimarc',
            marc_field  => '245a',
        },
        {
            name        => 'title',
            type        => 'string',
            facet       => 0,
            suggestible => undef,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '220',
        },
        {
            name        => 'uniform_title',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '240a',
        },
        {
            name        => 'title_wildcard',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => undef,
            marc_type   => 'marc21',
            marc_field  => '245',
        },
        {
            name        => 'sum_item_price',
            type        => 'sum',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '952g',
        },
        {
            name        => 'items_withdrawn_status',
            type        => 'boolean',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '9520',
        },
        {
            name        => 'local_classification',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '952o',
        },
        {
            name        => 'type_of_record',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => 'leader_/6',
        },
        {
            name        => 'type_of_record_and_bib_level',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => 'leader_/6-7',
        },
        {
            name        => 'ff7-00',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '007_/0',
        },
        {
            name        => 'issues',
            type        => 'sum',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '952l',
        },
        {
            name        => 'copydate',
            type        => 'year',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '260c',
        },
        {
            name        => 'date-of-publication',
            type        => 'year',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '008_/7-10',
        },
        {
            name        => 'subject',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '650(avxyz)',
        },
        {
            name        => 'note',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 1,
            marc_type   => 'marc21',
            marc_field  => '522a',
        },
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->mock(
        '_foreach_mapping',
        sub {
            my ( $self, $sub ) = @_;

            foreach my $map (@mappings) {
                $sub->(
                    $map->{name},
                    $map->{type},
                    $map->{facet},
                    $map->{suggestible},
                    $map->{sort},
                    $map->{searchable},
                    $map->{filter},
                    $map->{marc_type},
                    $map->{marc_field}
                );
            }
        }
    );

    my $see = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );

    my $callno      = 'ABC123';
    my $callno2     = 'ABC456';
    my $long_callno = '1234567890' x 30;

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new( '001', '123' ),
        MARC::Field->new( '007', 'ku' ),
        MARC::Field->new( '008', '901111s1962 xxk|||| |00| ||eng c' ),
        MARC::Field->new( '020', '', '',  a => '1-56619-909-3' ),
        MARC::Field->new( '100', '', '',  a => 'Author 1' ),
        MARC::Field->new( '110', '', '',  a => 'Corp Author' ),
        MARC::Field->new( '210', '', '',  a => 'Title 1' ),
        MARC::Field->new( '240', '', '4', a => 'The uniform title with nonfiling indicator' ),
        MARC::Field->new( '245', '', '',  a => 'Title:',     b => 'first record' ),
        MARC::Field->new( '260', '', '',  a => 'New York :', b => 'Ace ,',      c => 'c1962' ),
        MARC::Field->new( '650', '', '',  a => 'Heading',    z => 'Geohead',    v => 'Formhead' ),
        MARC::Field->new( '650', '', '',  a => 'Heading',    x => 'Gensubhead', z => 'Geohead' ),
        MARC::Field->new( '999', '', '',  c => '1234567' ),

        # '  ' for testing trimming of white space in boolean value callback:
        MARC::Field->new( '952', '', '', 0 => '  ', g => '123.30', o => $callno,      l => 3 ),
        MARC::Field->new( '952', '', '', 0 => 0,    g => '127.20', o => $callno2,     l => 2 ),
        MARC::Field->new( '952', '', '', 0 => 1,    g => '0.00',   o => $long_callno, l => 1 ),
    );
    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->leader('     cam  22      a 4500');
    $marc_record_2->append_fields(
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
        MARC::Field->new( '100', '', '', a => 'Author 2' ),

        # MARC::Field->new('210', '', '', a => 'Title 2'),
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
        MARC::Field->new( '260', '', '', a => 'New York :', b => 'Ace ,', c => '1963-2003' ),
        MARC::Field->new( '999', '', '', c => '1234568' ),
        MARC::Field->new( '952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno ),
    );

    my $marc_record_3 = MARC::Record->new();
    $marc_record_3->leader('     cam  22      a 4500');
    $marc_record_3->append_fields(
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
        MARC::Field->new( '100', '', '', a => 'Author 2' ),

        # MARC::Field->new('210', '', '', a => 'Title 3'),
        # MARC::Field->new('245', '', '', a => 'Title: third record'),
        MARC::Field->new( '260', '', '', a => 'New York :', b => 'Ace ,', c => ' 89 ' ),
        MARC::Field->new( '999', '', '', c => '1234568' ),
        MARC::Field->new( '952', '', '', 0 => 1, g => 'string where should be numeric', o => $long_callno ),
    );

    my $marc_record_4 = MARC::Record->new();
    $marc_record_4->leader('     cam  22      a 4500');
    $marc_record_4->append_fields(
        MARC::Field->new( '008', '901111s19uu xxk|||| |00| ||eng c' ),
        MARC::Field->new( '100', '', '',  a => 'Author 2' ),
        MARC::Field->new( '245', '', '4', a => 'The Title\'s the thing :', b => 'fourth record' ),
        MARC::Field->new( '260', '', '',  a => 'New York :', b => 'Ace ,', c => ' 89 ' ),
        MARC::Field->new( '999', '', '',  c => '1234568' ),
    );

    my $records = [ $marc_record_1, $marc_record_2, $marc_record_3, $marc_record_4 ];

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first

    my $docs = $see->marc_records_to_documents($records);

    # First record:
    is( scalar @{$docs}, 4, 'Four records converted to documents' );

    is_deeply( $docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly' );

    is_deeply( $docs->[0]->{'ff7-00'}, ['k'], 'First record ff7-00 should be set correctly' );

    is( scalar @{ $docs->[0]->{author} }, 2, 'First document author field should contain two values' );
    is_deeply(
        $docs->[0]->{author}, [ 'Author 1', 'Corp Author' ],
        'First document author field should be set correctly'
    );

    is( scalar @{ $docs->[0]->{subject} }, 2, 'First document subject field should contain two values' );
    is_deeply(
        $docs->[0]->{subject}, [ 'Heading Geohead Formhead', 'Heading Gensubhead Geohead' ],
        'First document asubject field should be set correctly, record order preserved for grouped subfield mapping'
    );

    is( scalar @{ $docs->[0]->{author__sort} }, 1, 'First document author__sort field should have a single value' );
    is_deeply(
        $docs->[0]->{author__sort}, ['Author 1 Corp Author'],
        'First document author__sort field should be set correctly'
    );

    is( scalar @{ $docs->[0]->{title__sort} }, 1, 'First document title__sort field should have a single' );
    is_deeply(
        $docs->[0]->{title__sort}, ['Title: first record Title: first record'],
        'First document title__sort field should be set correctly'
    );

    is( scalar @{ $docs->[3]->{title__sort} }, 1, 'First document title__sort field should have a single' );
    is_deeply(
        $docs->[3]->{title__sort}, ['Title\'s the thing : fourth record The Title\'s the thing : fourth record'],
        'Fourth document title__sort field should be set correctly'
    );
    is_deeply(
        $docs->[3]->{'title-no-punctuation'},
        [ 'The Titles the thing ', 'Titles the thing ', 'fourth record', 'The Titles the thing  fourth record' ],
        'Fourth document title-no-punctuation field should be set correctly'
    );

    is( $docs->[0]->{issues},       6, 'Issues field should be sum of the issues for each item' );
    is( $docs->[0]->{issues__sort}, 6, 'Issues sort field should also be a sum of the issues' );

    is( scalar @{ $docs->[0]->{title_wildcard} }, 2, 'First document title_wildcard field should have two values' );
    is_deeply(
        $docs->[0]->{title_wildcard}, [ 'Title:', 'first record' ],
        'First document title_wildcard field should be set correctly'
    );

    is(
        scalar @{ $docs->[0]->{author__suggestion} }, 2,
        'First document author__suggestion field should contain two values'
    );
    is_deeply(
        $docs->[0]->{author__suggestion},
        [
            { 'input' => 'Author 1' },
            { 'input' => 'Corp Author' }
        ],
        'First document author__suggestion field should be set correctly'
    );

    is(
        scalar @{ $docs->[0]->{title__suggestion} }, 3,
        'First document title__suggestion field should contain three values'
    );
    is_deeply(
        $docs->[0]->{title__suggestion},
        [
            { 'input' => 'Title:' },
            { 'input' => 'first record' },
            { 'input' => 'Title: first record' }
        ],
        'First document title__suggestion field should be set correctly'
    );

    ok( !( defined $docs->[0]->{title__facet} ), 'First document should have no title__facet field' );

    is( scalar @{ $docs->[0]->{author__facet} }, 2, 'First document author__facet field should have two values' );
    is_deeply(
        $docs->[0]->{author__facet},
        [ 'Author 1', 'Corp Author' ],
        'First document author__facet field should be set correctly'
    );

    is(
        scalar @{ $docs->[0]->{items_withdrawn_status} }, 2,
        'First document items_withdrawn_status field should have two values'
    );
    is_deeply(
        $docs->[0]->{items_withdrawn_status},
        [ 'false', 'true' ],
        'First document items_withdrawn_status field should be set correctly'
    );

    is(
        $docs->[0]->{sum_item_price},
        '250.5',
        'First document sum_item_price field should be set correctly'
    );

    ok( defined $docs->[0]->{marc_data},   'First document marc_data field should be set' );
    ok( defined $docs->[0]->{marc_format}, 'First document marc_format field should be set' );
    is( $docs->[0]->{marc_format}, 'base64ISO2709', 'First document marc_format should be set correctly' );

    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );

    ok( $decoded_marc_record->isa('MARC::Record'), "base64ISO2709 record successfully decoded from result" );
    is(
        $decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(),
        "Decoded base64ISO2709 record has same data as original record"
    );

    is( scalar @{ $docs->[0]->{type_of_record} }, 1, 'First document type_of_record field should have one value' );
    is_deeply(
        $docs->[0]->{type_of_record},
        ['a'],
        'First document type_of_record field should be set correctly'
    );

    is(
        scalar @{ $docs->[0]->{type_of_record_and_bib_level} }, 1,
        'First document type_of_record_and_bib_level field should have one value'
    );
    is_deeply(
        $docs->[0]->{type_of_record_and_bib_level},
        ['am'],
        'First document type_of_record_and_bib_level field should be set correctly'
    );

    is( scalar @{ $docs->[0]->{isbn} }, 4, 'First document isbn field should contain four values' );
    is_deeply(
        $docs->[0]->{isbn}, [ '978-1-56619-909-4', '9781566199094', '1-56619-909-3', '1566199093' ],
        'First document isbn field should be set correctly'
    );

    is_deeply(
        $docs->[0]->{'local_classification'},
        [ $callno, $callno2, $long_callno ],
        'First document local_classification field should be set correctly'
    );

    # Nonfiling characters for sort fields
    is_deeply(
        $docs->[0]->{uniform_title},
        [ 'The uniform title with nonfiling indicator', 'uniform title with nonfiling indicator' ],
        'First document uniform_title field should contain the title verbatim and with four initial characters removed'
    );
    is_deeply(
        $docs->[0]->{uniform_title__sort},
        ['uniform title with nonfiling indicator'],
        'First document uniform_title__sort field should contain the title with the first four initial characters removed'
    );

    # Tests for 'year' type
    is(
        scalar @{ $docs->[0]->{'date-of-publication'} }, 1,
        'First document date-of-publication field should contain one value'
    );
    is_deeply(
        $docs->[0]->{'date-of-publication'}, ['1962'],
        'First document date-of-publication field should be set correctly'
    );

    is_deeply(
        $docs->[0]->{'copydate'},
        ['1962'],
        'First document copydate field should be set correctly'
    );

    # Second record:

    is( scalar @{ $docs->[1]->{author} }, 1, 'Second document author field should contain one value' );
    is_deeply( $docs->[1]->{author}, ['Author 2'], 'Second document author field should be set correctly' );

    is(
        scalar @{ $docs->[1]->{items_withdrawn_status} }, 1,
        'Second document items_withdrawn_status field should have one value'
    );
    is_deeply(
        $docs->[1]->{items_withdrawn_status},
        ['true'],
        'Second document items_withdrawn_status field should be set correctly'
    );

    is(
        $docs->[1]->{sum_item_price},
        0,
        'Second document sum_item_price field should be set correctly'
    );

    is_deeply(
        $docs->[1]->{local_classification__sort},
        [ substr( $long_callno, 0, 255 ) ],
        'Second document local_classification__sort field should be set correctly'
    );

    # Tests for 'year' type
    is_deeply(
        $docs->[1]->{'copydate'},
        [ '1963', '2003' ],
        'Second document copydate field should be set correctly'
    );
    is_deeply(
        $docs->[1]->{'date-of-publication'},
        ['1900'],
        'Second document date-of-publication field should be set correctly'
    );

    # Third record:

    is_deeply(
        $docs->[2]->{'copydate'},
        ['0890'],
        'Third document copydate field should be set correctly'
    );

    # Mappings marc_type:

    ok( !( defined $docs->[0]->{unimarc_title} ), "No mapping when marc_type doesn't match marc flavour" );

    # Marc serialization format fallback for records exceeding base64ISO2709 max record size

    my $large_marc_record = MARC::Record->new();
    $large_marc_record->leader('     cam  22      a 4500');

    $large_marc_record->append_fields(
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large record' ),
        MARC::Field->new( '999', '', '', c => '1234567' ),
    );

    my $item_field = MARC::Field->new(
        '952', '', '', o => '123456789123456789123456789', p => '123456789',
        z => Encode::decode( 'UTF-8', 'To naprawdę bardzo długa notatka. Myślę, że będzie sprawiać kłopoty.' )
    );
    my $items_count = 1638;
    while ( --$items_count ) {
        $large_marc_record->append_fields($item_field);
    }

    $docs = $see->marc_records_to_documents( [$large_marc_record] );

    is(
        $docs->[0]->{marc_format}, 'MARCXML',
        'For record exceeding max record size marc_format should be set correctly'
    );

    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );

    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
    is(
        $decoded_marc_record->as_xml_record(), $large_marc_record->as_xml_record(),
        "Decoded MARCXML record has same data as original record"
    );

    push @mappings, {
        name        => 'title',
        type        => 'string',
        facet       => 0,
        suggestible => 1,
        sort        => 1,
        marc_type   => 'marc21',
        marc_field  => '245((ab)ab',
    };

    my $exception = try {
        $see->marc_records_to_documents($records);
    } catch {
        return $_;
    };

    ok( defined $exception, "Exception has been thrown when processing mapping with unmatched opening parenthesis" );
    ok( $exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class" );
    ok( $exception->message =~ /Unmatched opening parenthesis/, "Exception has the correct message" );

    pop @mappings;
    push @mappings, {
        name        => 'title',
        type        => 'string',
        facet       => 0,
        suggestible => 1,
        sort        => 1,
        marc_type   => 'marc21',
        marc_field  => '245(ab))ab',
    };

    $exception = try {
        $see->marc_records_to_documents($records);
    } catch {
        return $_;
    };

    ok( defined $exception, "Exception has been thrown when processing mapping with unmatched closing parenthesis" );
    ok( $exception->isa("Koha::Exceptions::Elasticsearch::MARCFieldExprParseError"), "Exception is of correct class" );
    ok( $exception->message =~ /Unmatched closing parenthesis/, "Exception has the correct message" );

    pop @mappings;
    my $marc_record_with_blank_field = MARC::Record->new();
    $marc_record_with_blank_field->leader('     cam  22      a 4500');

    $marc_record_with_blank_field->append_fields(
        MARC::Field->new( '100', '', '', a => '' ),
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'large record' ),
        MARC::Field->new( '999', '', '', c => '1234567' ),
    );
    $docs = $see->marc_records_to_documents( [$marc_record_with_blank_field] );
    is_deeply( $docs->[0]->{author},             [], 'No value placed into field if mapped marc field is blank' );
    is_deeply( $docs->[0]->{author__suggestion}, [], 'No value placed into suggestion if mapped marc field is blank' );

    my $marc_record_with_large_field = MARC::Record->new();
    $marc_record_with_large_field->leader('     cam  22      a 4500');

    my $xs = 'X' x 8191;
    my $ys = 'Y' x 8191;
    my $zs = 'Z' x 8191 . 'W';    # one extra character so it needs splitting

    $marc_record_with_large_field->append_fields(
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'record with large field' ),
        MARC::Field->new( '500', '', '', a => 'X' x 15000 ),
        MARC::Field->new( '522', '', '', a => "$xs $ys $zs" ),
        MARC::Field->new( '999', '', '', c => '1234567' ),
    );

    warning_is {
        $docs = $see->marc_records_to_documents( [$marc_record_with_large_field] );
    }
    "Warnings encountered while roundtripping a MARC record to/from USMARC. Failing over to MARCXML.";

    subtest '_process_mappings() split tests' => sub {

        plan tests => 4;

        my $note_indexes = $docs->[0]->{note};

        is( $note_indexes->[0], $xs,                    'First chunk split using the space' );
        is( $note_indexes->[1], $ys,                    'Second chunk split using the space' );
        is( $note_indexes->[2], substr( $zs, 0, 8191 ), 'Third chunk is forced to split' );
        is( $note_indexes->[3], substr( $zs, 8191, 1 ), 'Fourth chunk is just the remaining char' );
    };

    is( $docs->[0]->{marc_format}, 'MARCXML', 'For record with large field marc_format should be set correctly' );

    $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );

    ok( $decoded_marc_record->isa('MARC::Record'), "MARCXML record successfully decoded from result" );
    is(
        $decoded_marc_record->as_xml_record(), $marc_record_with_large_field->as_xml_record(),
        "Decoded MARCXML record has same data as original record"
    );

};

subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents_array () tests' => sub {

    plan tests => 5;

    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'ARRAY' );

    my @mappings = (
        {
            name        => 'control_number',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            sort        => undef,
            searchable  => 1,
            marc_type   => 'marc21',
            marc_field  => '001',
        }
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->mock(
        '_foreach_mapping',
        sub {
            my ( $self, $sub ) = @_;

            foreach my $map (@mappings) {
                $sub->(
                    $map->{name},
                    $map->{type},
                    $map->{facet},
                    $map->{suggestible},
                    $map->{sort},
                    $map->{searchable},
                    $map->{filter},
                    $map->{marc_type},
                    $map->{marc_field}
                );
            }
        }
    );

    my $see = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new( '001', '123' ),
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
        MARC::Field->new( '100', '', '', a => 'Author 1' ),
        MARC::Field->new( '110', '', '', a => 'Corp Author' ),
        MARC::Field->new( '210', '', '', a => 'Title 1' ),
        MARC::Field->new( '245', '', '', a => 'Title:', b => 'first record' ),
        MARC::Field->new( '999', '', '', c => '1234567' ),
    );
    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->leader('     cam  22      a 4500');
    $marc_record_2->append_fields(
        MARC::Field->new( '100', '', '', a => 'Author 2' ),

        # MARC::Field->new('210', '', '', a => 'Title 2'),
        # MARC::Field->new('245', '', '', a => 'Title: second record'),
        MARC::Field->new( '999', '', '', c => '1234568' ),
        MARC::Field->new( '952', '', '', 0 => 1, g => 'string where should be numeric' ),
    );
    my $records = [ $marc_record_1, $marc_record_2 ];

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first

    my $docs = $see->marc_records_to_documents($records);

    # First record:
    is( scalar @{$docs}, 2, 'Two records converted to documents' );

    is_deeply( $docs->[0]->{control_number}, ['123'], 'First record control number should be set correctly' );

    is( $docs->[0]->{marc_format}, 'ARRAY', 'First document marc_format should be set correctly' );

    my $decoded_marc_record = $see->decode_record_from_result( $docs->[0] );

    ok( $decoded_marc_record->isa('MARC::Record'), "ARRAY record successfully decoded from result" );
    is(
        $decoded_marc_record->as_usmarc(), $marc_record_1->as_usmarc(),
        "Decoded ARRAY record has same data as original record"
    );
};

subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents () authority tests' => sub {

    plan tests => 9;

    t::lib::Mocks::mock_preference( 'marcflavour',             'MARC21' );
    t::lib::Mocks::mock_preference( 'ElasticsearchMARCFormat', 'base64ISO2709' );

    my $builder = t::lib::TestBuilder->new;
    my $auth_type =
        $builder->build_object( { class => 'Koha::Authority::Types', value => { auth_tag_to_report => '150' } } );

    my @mappings = (
        {
            name        => 'match',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '150(aevxyz)',
        },
        {
            name        => 'match',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '185v',
        },
        {
            name        => 'subject-heading-thesaurus',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '008_/11',
        },
        {
            name        => 'subject-heading-thesaurus',
            type        => 'string',
            facet       => 0,
            suggestible => 0,
            searchable  => 1,
            sort        => 0,
            marc_type   => 'marc21',
            marc_field  => '040f',
        }
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->mock(
        '_foreach_mapping',
        sub {
            my ( $self, $sub ) = @_;

            foreach my $map (@mappings) {
                $sub->(
                    $map->{name},
                    $map->{type},
                    $map->{facet},
                    $map->{suggestible},
                    $map->{sort},
                    $map->{searchable},
                    $map->{filter},
                    $map->{marc_type},
                    $map->{marc_field}
                );
            }
        }
    );

    my $see = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::AUTHORITIES_INDEX } );
    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->append_fields(
        MARC::Field->new( '001', '123' ),
        MARC::Field->new( '007', 'ku' ),
        MARC::Field->new( '020', '', '', a => '1-56619-909-3' ),
        MARC::Field->new( '150', '', '', a => 'Subject', v => 'Genresubdiv', x => 'Generalsubdiv', z => 'Geosubdiv' ),
    );
    my $marc_record_2 = MARC::Record->new();
    $marc_record_2->append_fields(
        MARC::Field->new(
            '150', '', '', a => 'Subject', v => 'Genresubdiv', z => 'Geosubdiv', x => 'Generalsubdiv',
            e => 'wrongsubdiv'
        ),
    );
    my $marc_record_3 = MARC::Record->new();
    $marc_record_3->append_fields(
        MARC::Field->new( '185', '', '', v => 'Formsubdiv' ),
    );

    my $marc_record_4 = MARC::Record->new();
    $marc_record_4->append_fields(
        MARC::Field->new( '008', ' ' x 11 . 'z' . ' ' x 28 ),
        MARC::Field->new( '040', ' ', ' ', f => 'special_sauce' ),
        MARC::Field->new( '150', ' ', ' ', a => 'Philosophy' ),
    );

    my $marc_record_5 = MARC::Record->new();
    $marc_record_5->append_fields(
        MARC::Field->new( '008', ' ' x 11 . 'z' . ' ' x 28 ),
        MARC::Field->new( '150', ' ', ' ', a => 'Philosophy' ),
    );
    my $records = [ $marc_record_1, $marc_record_2, $marc_record_3, $marc_record_4, $marc_record_5 ];

    $see->get_elasticsearch_mappings(); #sort_fields will call this and use the actual db values unless we call it first

    my $docs = $see->marc_records_to_documents($records);

    is_deeply(
        ["Subject formsubdiv Genresubdiv generalsubdiv Generalsubdiv geographicsubdiv Geosubdiv"],
        $docs->[0]->{'match-heading'},
        "First record match-heading should contain the correctly formatted heading"
    );
    is_deeply(
        ["Subject formsubdiv Genresubdiv geographicsubdiv Geosubdiv generalsubdiv Generalsubdiv"],
        $docs->[1]->{'match-heading'},
        "Second record match-heading should contain the correctly formatted heading without wrong subfield"
    );
    is_deeply(
        ["Subject Genresubdiv Geosubdiv Generalsubdiv wrongsubdiv"],
        $docs->[1]->{'match'},
        "Second record heading should contain the subfields with record order retained"
    );
    ok( !exists $docs->[2]->{'match-heading'}, "No match heading defined for subdivision record" );
    is_deeply(
        ["Formsubdiv"],
        $docs->[2]->{'match'},
        "Third record heading should contain the subfield"
    );
    is(
        scalar( @{ $docs->[3]->{'subject-heading-thesaurus'} } ),
        1,
        "Only one subject-heading-thesaurus element for a record with 040 \$f"
    );
    is_deeply(
        $docs->[3]->{'subject-heading-thesaurus'}->[0],
        'special_sauce',
        "Fourth record's subject-heading-thesaurus taken from 040 \$f"
    );
    is(
        scalar( @{ $docs->[4]->{'subject-heading-thesaurus'} } ),
        1,
        "Just one subject-heading-thesaurus element for a record with 008/11 = 'z' and without 040 \$f"
    );
    is_deeply(
        $docs->[4]->{'subject-heading-thesaurus'}->[0],
        'z', "Fifth record's subject-heading-thesaurus taken from 008/11"
    );

};

subtest 'Koha::SearchEngine::Elasticsearch::marc_records_to_documents with IncludeSeeFromInSearches' => sub {

    plan tests => 4;

    t::lib::Mocks::mock_preference( 'marcflavour',              'MARC21' );
    t::lib::Mocks::mock_preference( 'IncludeSeeFromInSearches', '1' );
    my $dbh = C4::Context->dbh;

    my $builder   = t::lib::TestBuilder->new;
    my $auth_type = $builder->build_object(
        {
            class => 'Koha::Authority::Types',
            value => { auth_tag_to_report => '150' }
        }
    );
    my $authority_record = MARC::Record->new();
    $authority_record->append_fields(
        MARC::Field->new( 150, '', '', a => 'Foo' ),
        MARC::Field->new( 450, '', '', a => 'Bar' ),
    );
    $dbh->do(
        "INSERT INTO auth_header (datecreated,marcxml) values (NOW(),?)", undef,
        ( $authority_record->as_xml_record('MARC21') )
    );
    my $authid = $dbh->last_insert_id( undef, undef, 'auth_header', 'authid' );

    my @mappings = (
        {
            name        => 'subject',
            type        => 'string',
            facet       => 1,
            suggestible => 1,
            sort        => undef,
            searchable  => 1,
            marc_type   => 'marc21',
            marc_field  => '650a',
        }
    );

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->mock(
        '_foreach_mapping',
        sub {
            my ( $self, $sub ) = @_;

            foreach my $map (@mappings) {
                $sub->(
                    $map->{name},
                    $map->{type},
                    $map->{facet},
                    $map->{suggestible},
                    $map->{sort},
                    $map->{searchable},
                    $map->{filter},
                    $map->{marc_type},
                    $map->{marc_field}
                );
            }
        }
    );

    my $see = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );

    my $marc_record_1 = MARC::Record->new();
    $marc_record_1->leader('     cam  22      a 4500');
    $marc_record_1->append_fields(
        MARC::Field->new( '001', '123' ),
        MARC::Field->new( '245', '', '', a => 'Title' ),
        MARC::Field->new( '650', '', '', a => 'Foo', 9 => $authid ),
        MARC::Field->new( '999', '', '', c => '1234567' ),
    );

    # sort_fields will call this and use the actual db values unless we call it first
    $see->get_elasticsearch_mappings();

    my $docs = $see->marc_records_to_documents( [$marc_record_1] );

    is_deeply( $docs->[0]->{subject},        [ 'Foo', 'Bar' ], 'subject should include "See from"' );
    is_deeply( $docs->[0]->{subject__facet}, ['Foo'],          'subject__facet should not include "See from"' );
    is_deeply(
        $docs->[0]->{subject__suggestion}, [ { input => 'Foo' } ],
        'subject__suggestion should not include "See from"'
    );
    is_deeply( $docs->[0]->{subject__sort}, ['Foo'], 'subject__sort should not include "See from"' );
};

subtest 'marc_records_to_documents should set the "available" field' => sub {
    plan tests => 8;

    t::lib::Mocks::mock_preference( 'marcflavour', 'MARC21' );
    my $dbh = C4::Context->dbh;

    my $se = Test::MockModule->new('Koha::SearchEngine::Elasticsearch');
    $se->noop('_foreach_mapping');

    my $see = Koha::SearchEngine::Elasticsearch::Search->new(
        { index => $Koha::SearchEngine::Elasticsearch::BIBLIOS_INDEX } );

    # sort_fields will call this and use the actual db values unless we call it first
    $see->get_elasticsearch_mappings();

    my $builder       = t::lib::TestBuilder->new;
    my $biblio        = $builder->build_sample_biblio;
    my $marc_record_1 = $biblio->metadata->record;

    my $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \0, 'a biblio without items is not available' );

    my $item = $builder->build_sample_item(
        {
            biblionumber => $biblio->biblionumber,
        }
    );

    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \1, 'a biblio with one item that has no particular status is available' );

    $item->notforloan(1)->store();
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \1, 'a biblio with one item that is "notforloan" is available' );

    $item->set( { notforloan => 0, onloan => '2022-03-03' } )->store();
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \0, 'a biblio with one item that is on loan is not available' );

    $item->set( { onloan => undef, withdrawn => 1 } )->store();
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \1, 'a biblio with one item that is withdrawn is available' );

    $item->set( { withdrawn => 0, itemlost => 1 } )->store();
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \0, 'a biblio with one item that is lost is not available' );

    $item->set( { itemlost => 0, damaged => 1 } )->store();
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply( $docs->[0]->{available}, \1, 'a biblio with one item that is damaged is available' );

    my $item2 = $builder->build_sample_item(
        {
            biblionumber => $biblio->biblionumber,
        }
    );
    $docs = $see->marc_records_to_documents( [$marc_record_1] );
    is_deeply(
        $docs->[0]->{available}, \1,
        'a biblio with at least one item that has no particular status is available'
    );
};

$schema->storage->txn_rollback;
