Commit d4a1ecc5 authored by Jack Stupple's avatar Jack Stupple

create resources import instead of migrating them

parent 0c4a77f4
<?php
namespace App\Console\Commands;
use App\Language;
use App\Resource;
use App\ResourceCategory;
use Illuminate\Console\Command;
/**
* Class ImportResources
* @package App\Console\Commands
* @see https://laravel.com/docs/5.6/artisan
*/
class ImportResources extends Command
{
protected $signature = 'import:resources {csv_file_name}';
protected $description = 'Import resources into the API via a CSV file.';
protected $csv_file_name = null;
protected $csv_file_pathinfo = [];
protected $csv_headers = null;
protected $csv_data = [];
protected static $required_headers = [
'resource groups',
'category',
'title'
];
protected static $cached_langages = [];
protected static $cached_categories = [];
protected static $language_slug_replacement = [
'uk' => 'en'
];
public function handle()
{
ini_set('auto_detect_line_endings', true);
$this->csv_file_name = $this->argument('csv_file_name');
try {
// exception will be thrown if file is invalid
$this->csv_file_pathinfo = $this->checkFileProperties($this->csv_file_name);
} catch (\Exception $exception) {
$this->output->writeln("<error>Error encountered:</error> {$exception->getMessage()}");
return false;
}
list($this->csv_headers, $this->csv_data) = $this->getFileData($this->csv_file_name);
$this->checkHeaders($this->csv_headers); // throws exception if fails
// we now have the resources list
foreach ($this->csv_data as $csv_row_number => $_resource) {
$language = null;
$resource_link = null;
// add here cases for the different kinds of resources we are importing
if (!empty($_resource['uk asset'])) {
$language = $this->getLanguageFor('uk');
$resource_link = $_resource['uk asset'];
} else if (!empty($_resource['fr asset'])) {
$language = $this->getLanguageFor('fr');
$resource_link = $_resource['fr asset'];
} else if (!empty($_resource['de asset'])) {
$language = $this->getLanguageFor('de');
$resource_link = $_resource['de asset'];
} else if (!empty($_resource['no asset'])) {
$language = $this->getLanguageFor('no');
$resource_link = $_resource['no asset'];
}
// check for duplicates
if ($this->isDuplicateResource($language->id, $resource_link)) {
$this->output->writeln("<comment>Skipped row due to duplicate:</comment> {$csv_row_number}");
continue;
}
$resource = new Resource();
$resource->language_id = $language->id;
$resource->name = $_resource['title'];
$resource->link = $resource_link;
if ($resource->save() !== true) {
$this->output->writeln("<error>Failed to save row:</error> {$csv_row_number}");
}
// attach it to the category (this is a belongsToMany relationship. A resource can be in multiple categories, intended for admin)
$resource->categories()->attach($this->getResourceCategory($_resource['category'], $_resource['resource groups']));
}
}
protected function getResourceCategory($category, $parent_category)
{
if (isset(static::$cached_categories[$category])) {
return static::$cached_categories[$category];
}
$resource_category = ResourceCategory::firstOrCreate([
'name' => $category,
'parent_category' => $parent_category
]);
if (!$resource_category->exists()) {
$resource_category->save();
}
static::$cached_categories[$category] = $resource_category;
return $resource_category;
}
protected function isDuplicateResource($language_id, $resource_link)
{
return Resource::where('link', $resource_link)->where('language_id', $language_id)->count();
}
protected function getLanguageFor($language_slug)
{
// convert uk to en just incase it comes up
if (in_array($language_slug, array_keys(static::$language_slug_replacement))) {
$language_slug = static::$language_slug_replacement[$language_slug];
}
if (isset(static::$cached_langages[$language_slug])) {
return static::$cached_langages[$language_slug];
}
$language = Language::where('slug', $language_slug)->firstOrFail();
static::$cached_langages[$language_slug] = $language;
return $language;
}
protected function checkHeaders($csv_headers)
{
foreach (static::$required_headers as $required_header) {
if (in_array($required_header, $csv_headers) === false) {
throw new \Exception ("Missing required header in CSV: {$required_header}");
}
}
}
protected function getFileData($csv_file_name)
{
$file_handle = fopen($csv_file_name, 'r');
$csv_headers = null;
$csv_data = [];
$row_number = 0;
$empty_rows = 0; // if 3 empty rows are encountered we just break the loop- assumed end of data
while (($csv_row = fgetcsv($file_handle)) !== false) {
$row_number++; // the header is also a row in the CSV
if ($csv_headers === null) {
$_csv_headers = [];
foreach ($csv_row as $header) {
$_csv_headers[] = trim(strtolower($header));
}
$csv_headers = $_csv_headers;
unset($_csv_headers);
continue;
}
// skip empty rows
if (array_filter(array_unique($csv_row)) === []) {
$empty_rows++;
if ($empty_rows >= 3) {
// assume we have reached the end of the data
$this->output->writeln("<comment>Assumed end of data:</comment> {$row_number}");
break;
} else {
$this->output->writeln("<comment>Skip empty row:</comment> {$row_number}");
continue;
}
}
$_csv_data = [];
foreach ($csv_headers as $header_index => $header) {
$_csv_data[$header] = trim($csv_row[$header_index]); // we trim to ensure no additional spaces
// ensure the data is expected - primarily for the URL's
$validation_method = camel_case('validate ' . $header);
if (method_exists($this, $validation_method) && !empty($_csv_data[$header])) {
if (!$this->$validation_method($_csv_data[$header])) {
throw new \Exception("Issue with {$header} on row: {$row_number}");
}
}
}
$csv_data[$row_number] = $_csv_data; // add the row to the data with same row number as that in the CSV
unset($_csv_data);
}
return [$csv_headers, $csv_data];
}
protected function checkFileProperties($csv_file_name)
{
if (file_exists($csv_file_name) === false) {
throw new \Exception('File does not exist');
}
$csv_file_pathinfo = pathinfo($csv_file_name);
// extension is only defined if one is provided, eg /etc/hosts will have no extension
if (!isset($csv_file_pathinfo['extension']) || $csv_file_pathinfo['extension'] !== 'csv') {
throw new \Exception('File is not CSV');
}
return $csv_file_pathinfo;
}
protected function validateUkAsset($uk_asset)
{
return $this->validateUrl($uk_asset);
}
protected function validateFrAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateDeAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateNoAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateUrl($asset)
{
return filter_var($asset, FILTER_VALIDATE_URL);
}
}
\ No newline at end of file
......@@ -2,6 +2,7 @@
namespace App\Console;
use App\Console\Commands\ImportResources;
use Illuminate\Console\Scheduling\Schedule;
use Laravel\Lumen\Console\Kernel as ConsoleKernel;
......@@ -13,7 +14,7 @@ class Kernel extends ConsoleKernel
* @var array
*/
protected $commands = [
//
ImportResources::class
];
/**
......
......@@ -6,6 +6,11 @@ use Illuminate\Database\Eloquent\Model;
class ResourceCategory extends Model
{
protected $fillable = [
'name',
'parent_category'
];
public function resources()
{
return $this->hasMany(\App\Resource::class);
......
......@@ -122,34 +122,6 @@ class Initial extends Migration
],
];
static $resource_categories = [
["Accounts", "banking"],
["Bonds", "investments"],
["Borrowing money", "loans"],
["Budgeting", "budgeting"],
["Car repayments", "loans"],
["Cash", "banking"],
["Credit cards", "loans"],
["Credit options", "loans"],
["Debit cards", "banking"],
["Interest", "loans"],
["Interest rates", "loans"],
["Loans", "loans"],
["Payday loans", "loans"],
["Phone deals", "loans"],
["Phone Depreciation", "investments"],
["Saving for a car", "banking"],
["Savings", "banking"],
["Startups", "investments"],
["Store credit", "loans"],
["Prepaid Cards", "banking"],
["Packaged Accounts", "banking"],
["Mortgages", "loans"],
["Retirement", "investments"]
];
static $resources = '/resources/data/resources.csv';
protected $created_at = null;
protected $language = null;
......@@ -397,38 +369,6 @@ class Initial extends Migration
$blueprint->timestamps();
}
);
foreach (static::$resource_categories as $_resource_category) {
list($name, $parent_category) = $_resource_category;
$resource_category = new \App\ResourceCategory;
$resource_category->name = $name;
$resource_category->parent_category = $parent_category;
$resource_category->save();
}
$resources_csv_handle = fopen(APP_ROOT . static::$resources, 'r');
$headers = false;
while (($_resource = fgetcsv($resources_csv_handle)) !== false) {
if (empty($headers)) {
$headers = true;
continue;
}
$category = \App\ResourceCategory::where('name', 'LIKE', $_resource[1])->first();
if (!$category) {
exit ('category missing:' . $_resource[1]);
}
$resource = new \App\Resource;
$resource->language_id = $this->language->id;
$resource->name = $_resource[2];
$resource->link = $_resource[3];
$resource->save();
$resource->categories()->syncWithoutDetaching($category);
}
}
/**
......
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment