<?php

verifyCsrfGetToken();

require dirname(__FILE__) . '/../structure.php';
require dirname(__FILE__) . '/../inc/importlib.php';

@ob_end_flush();
$status = 'FAIL';
output('<p class="information">' . $GLOBALS['I18N']->get('Reading emails from file ... '));
flush();
if (filesize($_SESSION['import_file']) > 50000) {
    @ini_set('memory_limit', memory_get_usage() + 50 * filesize($_SESSION['import_file']));
}
$email_list = file_get_contents($_SESSION['import_file']);
flush();
// Clean up email file
$email_list = trim($email_list);
$email_list = str_replace("\r", "\n", $email_list);
$email_list = str_replace("\n\r", "\n", $email_list);
$email_list = str_replace("\n\n", "\n", $email_list);

if ($_SESSION['import_record_delimiter'] != "\n") {
    $email_list = str_replace($_SESSION['import_record_delimiter'], "\n", $email_list);
};

// Split file/emails into array
$email_list = explode("\n", $email_list); //WARNING the file contents get replace by an array
output(sprintf('..' . $GLOBALS['I18N']->get('ok, %d lines') . '</p>', count($email_list)));
$header = array_shift($email_list);
$header = str_replace('"', '', $header);
$total = count($email_list);
$headers = explode($_SESSION['import_field_delimiter'], $header);
$headers = array_unique($headers);
$_SESSION['columnnames'] = $headers;

### show progress and adjust working space
if (count($email_list)) {
    $import_field_delimiter = $_SESSION['import_field_delimiter'];
    if (count($email_list) > 300 && !$_SESSION['test_import']) {
        # this is a possibly a time consuming process, so show a progress bar
        print '<script language="Javascript" type="text/javascript"> document.write(progressmeter); start();</script>';
        flush();
        # increase the memory to make sure we are not running out
        #    $mem = sizeof($email_list);
        ini_set('memory_limit', '32M');
    }

    ### store the chosen mappings in the $system_attribute_mapping list
    # print "A: ".sizeof($import_attribute);
    foreach ($system_attributes as $key => $val) {
        if (isset($_SESSION['systemindex'][$key])) {
            $system_attribute_mapping[$key] = $_SESSION['systemindex'][$key];
        }
    }

    ### Parse the lines into records
    #  print "<br/>Loading emails .. ";
    flush();
    $count = array();
    $count['email_add'] = 0;
    $count['exist'] = 0;
    $count['list_add'] = 0;
    $count['group_add'] = 0;
    $count['foundblacklisted'] = 0;
    $c = 1;
    $count['invalid_email'] = 0;
    $num_lists = count($_SESSION['lists']);
    $total = count($email_list);
    $cnt = 0;
    $count['emailmatch'] = 0;
    $count['fkeymatch'] = 0;
    $count['dataupdate'] = 0;
    $count['duplicate'] = 0;
    $additional_emails = 0;
    foreach ($email_list as $line) {
        set_time_limit(60);
        # will contain attributes to store / change
        $user = array();
        # get rid of text delimiters generally added by spreadsheet apps
        $line = str_replace('"', '', $line);

        $values = explode($_SESSION['import_field_delimiter'], $line);
        $system_values = array();
        foreach ($system_attribute_mapping as $column => $index) {
            #   print '<br/>'.$column . ' = '. $values[$index];
            if (!empty($values[$index])) {
                $system_values[$column] = $values[$index];
            } else {
                $system_values[$column] = '';
            }
        }
        ## Check completeness
        $index = clean($system_values['email']);
        $invalid = 0;
        if (!$index) {
            if ($_SESSION['show_warnings']) {
                Warn($GLOBALS['I18N']->get('Record has no email') .
                    ": $c -> $line");
            }
            $index = $GLOBALS['I18N']->get('Invalid Email') . " $c";
            $system_values['email'] = $_SESSION['assign_invalid'];
            $invalid = 1;
            ++$count['invalid_email'];
        }

        //print ("<pre>" . var_dump($_SESSION["import_attribute"]) . "</pre>"); // debug
        //    dbg('_SESSION["import_attribute"',$_SESSION["import_attribute"]); //debug
        if (count($values) != (count($_SESSION['import_attribute']) + count($system_attributes) - count($unused_systemattr)) && !empty($_SESSION['test_import']) && !empty($_SESSION['show_warnings'])) {
            Warn('Record has more values than header indicated (' .
                count($values) . '!=' .
                (count($_SESSION['import_attribute']) + count($system_attributes) - count($unused_systemattr)) .
                "), this may cause trouble: $index");
        }
        if (!$invalid || ($invalid && $_SESSION['omit_invalid'] != 'yes')) {
            $user['systemvalues'] = $system_values;
            reset($_SESSION['import_attribute']);
            $replace = array();
            while (list($key, $val) = each($_SESSION['import_attribute'])) {
                if (!empty($values[$val['index']])) {
                    $user[$val['index']] = addslashes($values[$val['index']]);
                    $replace[$key] = addslashes($values[$val['index']]);
                }
            }
        } else {
            # Warn("Omitting invalid one: $email");
        }
        $user['systemvalues']['email'] = parsePlaceHolders($system_values['email'],
            array_merge($replace, $system_values, array(
                'number' => $c,
            )));
        $user['systemvalues']['email'] = cleanEmail($user['systemvalues']['email']);
        ++$c;
        if (!isset($user['systemvalues']['htmlemail'])) {
            $user['systemvalues']['htmlemail'] = 1;
        }
        if ($_SESSION['test_import']) {

#      var_dump($user["systemvalues"]);exit;
            $html = '';
            foreach ($user['systemvalues'] as $column => $value) {
                if (strpos($column, 'grouptype_') === 0) {
                    if (isset($system_attributes[$column])) {
                        $column = $system_attributes[$column];
                    }
                }
                if (!empty($column)) {
                    if ($value) {
                        $html .= "$column -> $value<br/>\n";
                    } else {
                        $html .= "$column -> " . $GLOBALS['I18N']->get('clear value') . "<br/>\n";
                    }
                }
            }
            #  var_dump($_SESSION["systemindex"]);

            reset($_SESSION['import_attribute']);
            foreach ($_SESSION['import_attribute'] as $column => $item) {
                if (!empty($user[$item['index']])) {
                    if ($item['record'] == 'new') {
                        $html .= ' ' . $GLOBALS['I18N']->get('New Attribute') . ': ' . $item['column'];
                    } elseif ($item['record'] == 'skip') {
                        # forget about it
                        $html .= ' ' . $GLOBALS['I18N']->get('Skip value') . ' ' . $column . ': ';
                    } elseif ($item['record'] != 'system') {
                        $html .= $attributes[$item['record']];
#            var_dump($attributes[$item['record']]);
                    } else {
                        $html .= $item['column'];
                    }
                    $html .= ' -> ' . $user[$item['index']] . '<br/>';
                }
            }
            if ($html) {
                print '<blockquote>' . $html . '</blockquote><hr />';
            }
        } else { // not test
            @ob_end_flush();
            if ($cnt % 5 == 0) {
                print '<script type="text/javascript">
        var parentJQuery = window.parent.jQuery;
        parentJQuery("#progressbar").updateProgress("' . $cnt . ',' . $total . '");
        </script>';
                flush();
            }
            ++$cnt;

            if (!$invalid || ($invalid && $_SESSION['omit_invalid'] != 'yes')) {
                # do import
                ## create new attributes
                foreach ($_SESSION['import_attribute'] as $column => $item) {
                    if ($item['record'] == 'new') {
                        Sql_Query(sprintf('insert into %s (name,type) values("%s","textline")', $tables['attribute'],
                            addslashes($column)));
                        $attid = Sql_Insert_id();
                        Sql_Query(sprintf('update %s set tablename = "attr%d" where id = %d', $tables['attribute'],
                            $attid, $attid));
                        Sql_Query('create table ' . $GLOBALS['table_prefix'] . 'listattr_attr' . $attid . '
                        (id integer not null primary key auto_increment, name varchar(255) unique,
                        listorder integer default 0)');
                        $_SESSION['import_attribute'][$column]['record'] = $attid;
                    }
                }
                $new = 0;
                if (!empty($user['systemvalues']['foreignkey'])) {
                    //         dbg('Importing on FK '.$user["systemvalues"]["foreignkey"].' email :'.$user["systemvalues"]["email"]);
                    $result = Sql_query(sprintf('select id,uniqid from %s where foreignkey = "%s"', $tables['user'],
                        $user['systemvalues']['foreignkey']));
                    # print "<br/>Using foreign key for matching: ".$user["systemvalues"]["foreign key"];
                    ++$count['fkeymatch'];
                    $exists = Sql_Affected_Rows();
                    $existing_user = Sql_fetch_array($result);
                    # check whether the email will clash
                    $clashcheck = Sql_Fetch_Array_Query(sprintf('select id,foreignkey,uniqid from %s
                    where email = "%s"', $tables['user'], $user['systemvalues']['email']));
                    if (!empty($clashcheck['id']) && $clashcheck['id'] != $existing_user['id']) {
                        #https://mantis.phplist.org/view.php?id=17752
                        # if the existing record does not have an FK, we treat it as an update, matched on email
                        if (empty($clashcheck['foreignkey'])) {
                            ++$count['emailmatch'];
                            --$count['fkeymatch'];
                            $exists = 1;
                            $existing_user = $clashcheck;
                        } else {
                            ++$count['duplicate'];
                            $notduplicate = 0;
                            $c = 0;
                            while (!$notduplicate) {
                                ++$c;
                                $req = Sql_Query(sprintf('select id from %s where email = "%s"', $tables['user'],
                                    $GLOBALS['I18N']->get('duplicate') .
                                    "$c " . $user['systemvalues']['email']));
                                $notduplicate = !Sql_Affected_Rows();
                            }
                            if (!$_SESSION['retainold']) {
                                Sql_Query(sprintf('update %s set email = "%s" where email = "%s"', $tables['user'],
                                    "duplicate$c " .
                                    $user['systemvalues']['email'], $user['systemvalues']['email']));
                                addUserHistory("duplicate$c " . $user['systemvalues']['email'], 'Duplication clash ',
                                    ' User marked duplicate email after clash with imported record');
                            } else {
                                if ($_SESSION['show_warnings']) {
                                    print Warn($GLOBALS['I18N']->get('Duplicate Email') . ' ' . $user['systemvalues']['email'] . $GLOBALS['I18N']->get(' user imported as ') . '&quot;' . $GLOBALS['I18N']->get('duplicate') . "$c " . $user['systemvalues']['email'] . '&quot;');
                                }
                                $user['systemvalues']['email'] = $GLOBALS['I18N']->get('duplicate') . "$c " . $user['systemvalues']['email'];
                            }
                        }
                    }
                } else {
                    dbg('Importing on email ' . $user['systemvalues']['email']);
                    $result = Sql_query(sprintf('select id,uniqid from %s where email = "%s"', $tables['user'],
                        $user['systemvalues']['email']));
                    # print "<br/>Using email for matching: ".$user["systemvalues"]["email"];
                    ++$count['emailmatch'];
                    $exists = Sql_Affected_Rows();
                    $existing_user = Sql_fetch_array($result);
                }
                if ($exists) {
                    // User exist, remember some values to add them to the lists
                    ++$count['exist'];
                    $userid = $existing_user['id'];
                    $uniqid = $existing_user['uniqid'];
                } else {
                    // user does not exist
                    $new = 1;
                    // Create unique number
                    mt_srand((double)microtime() * 1000000);
                    $randval = mt_rand();
                    # this is very time consuming when importing loads of users as it does a lookup
                    # needs speeding up if possible
                    $uniqid = getUniqid();
                    $confirmed = $_SESSION['notify'] != 'yes' && !preg_match('/Invalid Email/i', $index);

                    $query = sprintf('INSERT INTO %s (email,entered,confirmed,uniqid,htmlemail)
                    values("%s",now(),%d,"%s",1)', $tables['user'], $user['systemvalues']['email'], $confirmed,
                        $uniqid);
                    $result = Sql_query($query, 1);
                    $userid = Sql_insert_id();
                    if (!$userid) {
                        # no id returned, so it must have been a duplicate entry
                        if ($_SESSION['show_warnings']) {
                            print Warn($GLOBALS['I18N']->get('Duplicate Email') . ' ' . $user['systemvalues']['email']);
                        }
                        $c = 0;
                        while (!$userid) {
                            ++$c;
                            $query = sprintf('INSERT INTO %s (email,entered,confirmed,uniqid,htmlemail)
                            values("%s",now(),%d,"%s",1)', $tables['user'], $user['systemvalues']['email'] .
                                " ($c)", 0, $uniqid);
                            $result = Sql_query($query, 1);
                            $userid = Sql_insert_id();
                        }
                        $user['systemvalues']['email'] = $user['systemvalues']['email'] . " ($c)";
                    }

                    ++$count['email_add'];
                    $some = 1;
                }

                reset($_SESSION['import_attribute']);
                #   var_dump($_SESSION);exit;
                if ($new || (!$new && $_SESSION['overwrite'] == 'yes')) {
                    $query = '';
                    ++$count['dataupdate'];
                    $old_data = Sql_Fetch_Array_Query(sprintf('select * from %s where id = %d', $tables['user'],
                        $userid));
                    $old_data = array_merge($old_data, getUserAttributeValues('', $userid));
                    $history_entry = $GLOBALS['admin_scheme'] . '://' . getConfig('website') . $GLOBALS['adminpages'] . '/?page=user&id=' . $userid . "\n\n";
                    foreach ($user['systemvalues'] as $column => $value) {
                        if (!empty($column)) { # && !empty($value)) {
                            if ($column == 'groupmapping' || strpos($column, 'grouptype_') === 0) {
                                ## specifically request this group, so that it doesn't interfere with the "groups" which are the ones
                                ## submitted in the form

                                if (strpos($column, 'grouptype_') === 0) {
                                    list($tmp, $type) = explode('_', $column);
                                } else {
                                    $type = $_SESSION['grouptype'];
                                }
                                $type = sprintf('%d', $type);
                                ## verify the type is set
                                if (!in_array($type, array_keys($GLOBALS['config']['usergroup_types']))) {
                                    Warn('Invalid group membership type' . $type);
                                    dbg($type, 'Type not found');
                                }

                                $columnGroups = explode(',', $value);
                                foreach ($columnGroups as $sGroup) {
                                    $sGroup = trim($sGroup);
                                    $groupIdReq = Sql_Fetch_Row_Query(sprintf('select id from groups where name = "%s"',
                                        $sGroup));
                                    if (empty($groupIdReq[0])) {
                                        Sql_Query(sprintf('insert into groups (name) values("%s")', $sGroup));
                                        Warn("Group $sGroup added");
                                        $groupIdReq[0] = Sql_Insert_id();
                                    }
                                    dbg('Adding to group ' . $sGroup . ' with type ' . $GLOBALS['config']['usergroup_types'][$type]);
                                    ## @@ this may cause problems on not-upgraded DBs
                                    Sql_Query(sprintf('replace into user_group (userid,groupid,type) values(%d,%d,%d)',
                                        $userid, $groupIdReq[0], $type));
                                }
                            } else {
                                $query .= sprintf('%s = "%s",', $column, $value);
                            }
                        }
                    }
                    if ($query) {
                        $query = substr($query, 0, -1);
                        # this may cause a duplicate error on email, so add ignore
                        Sql_Query("update ignore {$tables['user']} set $query where id = $userid");
                    }
                    foreach ($_SESSION['import_attribute'] as $item) {
                        if (isset($user[$item['index']]) && is_numeric($item['record']) && strpos($item['record'],
                                'grouptype_') !== 0
                        ) {
                            $attribute_index = $item['record'];
                            $uservalue = $user[$item['index']];
                            # check whether this is a textline or a selectable item
                            $att = Sql_Fetch_Row_Query('select type,tablename,name from ' . $tables['attribute'] . " where id = $attribute_index"); ////
                            switch ($att[0]) {
                                case 'select' :
                                case 'radio' :
                                    $val = Sql_Query("select id from $table_prefix" . "listattr_$att[1] where name = \"$uservalue\"");
                                    # if we do not have this value add it
                                    if (!Sql_Affected_Rows()) {
                                        Sql_Query("insert into $table_prefix" . "listattr_$att[1] (name) values(\"$uservalue\")");
                                        Warn("Value $uservalue added to attribute $att[2]");
                                        $user_att_value = Sql_Insert_Id();
                                    } else {
                                        $d = Sql_Fetch_Row($val);
                                        $user_att_value = $d[0];
                                    }
                                    break;
                                case 'checkboxgroup':
                                    $values = explode(',', $uservalue);
                                    $valueIds = array();
                                    foreach ($values as $importValue) {
                                        $val = Sql_Query("select id from $table_prefix" . "listattr_$att[1] where name = \"$importValue\"");
                                        # if we do not have this value add it
                                        if (!Sql_Affected_Rows()) {
                                            Sql_Query("insert into $table_prefix" . "listattr_$att[1] (name) values(\"$importValue\")");
                                            Warn("Value $importValue added to attribute $att[2]");
                                            $valueIds[] = Sql_Insert_Id();
                                        } else {
                                            $d = Sql_Fetch_Row($val);
                                            $valueIds[] = $d[0];
                                        }
                                    }
                                    $user_att_value = implode(',', $valueIds);
                                    break;
                                case 'checkbox' :
                                    $uservalue = trim($uservalue);
                                    #print $uservalue;exit;
                                    if (!empty($uservalue) && $uservalue != 'off') {
                                        $user_att_value = 'on';
                                    } else {
                                        $user_att_value = '';
                                    }
                                    break;
                                case 'date' :
                                    #                $user_att_value = parseDate($uservalue);
                                    $user_att_value = $uservalue;
                                    break;
                                default :
                                    $user_att_value = $uservalue;
                                    break;
                            }

                            Sql_query(sprintf('replace into %s (attributeid,userid,value) values(%d,%d,"%s")',
                                $tables['user_attribute'], $attribute_index, $userid, $user_att_value));
                        } else {
                            if ($item['record'] != 'skip') {
                                # add an empty entry if none existed
                                Sql_Query(sprintf('insert ignore into %s (attributeid,userid,value) values(%d,%d,"")',
                                    $tables['user_attribute'], $item['record'], $userid));
                            }
                        }
                    }
                    $current_data = Sql_Fetch_Array_Query(sprintf('select * from %s where id = %d', $tables['user'],
                        $userid));
                    $current_data = array_merge($current_data, getUserAttributeValues('', $userid));
                    $information_changed = 0;
                    foreach ($current_data as $key => $val) {
                        if (!is_numeric($key)) {
                            if (isset($old_data[$key]) && $old_data[$key] != $val && $old_data[$key] && $key != 'password' && $key != 'modified') {
                                $information_changed = 1;
                                $history_entry .= "$key = $val\n*changed* from $old_data[$key]\n";
                            }
                        }
                    }
                    if (!$information_changed) {
                        $history_entry .= "\nNo user details changed";
                    }
                    addUserHistory($user['systemvalues']['email'], 'Import by ' . adminName(), $history_entry);
                }

                #add this user to the lists identified, except when they are blacklisted
                $isBlackListed = isBlackListed($user['systemvalues']['email']);
                if (!$isBlackListed && is_array($_SESSION['lists'])) {
                    reset($_SESSION['lists']);
                    $addition = 0;
                    $listoflists = '';
                    while (list($key, $listid) = each($_SESSION['lists'])) {
                        $query = 'replace INTO ' . $tables['listuser'] . " (userid,listid,entered) values($userid,$listid,now())";
                        $result = Sql_query($query, 1);
                        # if the affected rows is 2, the user was already subscribed
                        $addition = $addition || Sql_Affected_Rows() == 1;
                        $listoflists .= '  * ' . listName($key) . "\n";# $_SESSION["listname"][$key] . "\n";
                    }
                    if ($addition) {
                        ++$count['list_add'];
                    }
                    if (!TEST && $_SESSION['notify'] == 'yes' && $addition) {
                        $subscribemessage = str_replace('[LISTS]', $listoflists,
                            getUserConfig('subscribemessage', $userid));
                        if (function_exists('sendmail')) {
                            sendMail($user['systemvalues']['email'], getConfig('subscribesubject'), $subscribemessage,
                                system_messageheaders(), $envelope);
                            if (isset($_SESSION['throttle_import'])) {
                                sleep($_SESSION['throttle_import']);
                            }
                        }
                    }
                } elseif ($isBlackListed) {
                    ## mark blacklisted, just in case ##17288
                    Sql_Query(sprintf('update %s set blacklisted = 1 where id = %d', $tables['user'], $userid));
                    ++$count['foundblacklisted'];
                }
                if (!is_array($_SESSION['groups'])) {
                    $groups = array();
                } else {
                    $groups = $_SESSION['groups'];
                }
                if (isset($everyone_groupid) && !in_array($everyone_groupid, $groups)) {
                    array_push($groups, $everyone_groupid);
                }
                if (defined('IN_WEBBLER') && is_array($groups)) {
                    #add this user to the groups identified
                    reset($groups);
                    $groupaddition = 0;
                    while (list($key, $groupid) = each($groups)) {
                        if ($groupid) {
                            $query = sprintf('replace INTO user_group (userid,groupid,type) values(%d,%d,%d)', $userid,
                                $groupid, $_SESSION['grouptype']);
                            $result = Sql_query($query);
                            # if the affected rows is 2, the user was already subscribed
                            $groupaddition = $groupaddition || Sql_Affected_Rows() == 1;
                        }
                    }
                    if ($groupaddition) {
                        ++$count['group_add'];
                    }
                }
            }
        } // end else not test
        if ($_SESSION['test_import'] && $c > 50) {
            break;
        }
    }

    $report = '';
    if (empty($some) && !$count['list_add']) {
        $report .= '<br/>' . s('All the emails already exist in the database and are member of the lists');
    } else {
        $report .= '<br/>' . s('%d emails succesfully imported to the database and added to %d lists.',
                $count['email_add'], $num_lists);
        $report .= '<br/>' . s('%d emails subscribed to the lists', $count['list_add']);
        if ($count['exist']) {
            $report .= '<br/>' . s('%d emails already existed in the database', $count['exist']);
        }
    }
    if ($count['invalid_email']) {
        $report .= '<br/>' . s('%d Invalid Emails found.', $count['invalid_email']);
        if (!$_SESSION['omit_invalid']) {
            $report .= '<br/>' . s('These records were added, but the email has been made up from ') . $_SESSION['assign_invalid'];
        } else {
            $report .= '<br/>' . s('These records were deleted. Check your source and reimport the data. Duplicates will be identified.');
        }
    }
    if ($count['duplicate']) {
        $report .= '<br/>' . s('%d duplicate emails found.', $count['duplicate']);
    }
    if ($_SESSION['overwrite'] == 'yes') {
        $report .= '<br/>' . s('Subscriber data was updated for %d subscribers', $count['dataupdate']);
    }
    if ($count['foundblacklisted']) {
        $report .= '<br/>' . s('%d emails were on the blacklist and have not been added to the lists',
                $count['foundblacklisted']);
    }
    $report .= '<br/>' . s('%d subscribers were matched by foreign key, %d by email', $count['fkeymatch'],
            $count['emailmatch']);
    if (!$GLOBALS['commandline']) {
        print $report;
        if (function_exists('sendmail')) {
            sendMail(getConfig('admin_address'), $GLOBALS['I18N']->get('phplist Import Results'), $report);
        }
        if (function_exists('logevent')) {
            logEvent($report);
        }
        foreach ($GLOBALS['plugins'] as $pluginname => $plugin) {
            $plugin->importReport($report);
        }
    } else {
        output($report);
    }
    $htmlupdate = $report . '<br/>' . PageLinkButton('import2', s('Import some more emails'));
    $htmlupdate = str_replace("'", "\'", $htmlupdate);

    clearImport();
    $status = '<script type="text/javascript">
      var parentJQuery = window.parent.jQuery;
      parentJQuery("#progressbar").progressbar("destroy");
      parentJQuery("#busyimage").hide();
      parentJQuery("#progresscount").html(\'' . $htmlupdate . '\');
      </script>';
}
