Commit d4a1ecc5 authored by Jack Stupple's avatar Jack Stupple

create resources import instead of migrating them

parent 0c4a77f4
<?php
namespace App\Console\Commands;
use App\Language;
use App\Resource;
use App\ResourceCategory;
use Illuminate\Console\Command;
/**
* Class ImportResources
* @package App\Console\Commands
* @see https://laravel.com/docs/5.6/artisan
*/
class ImportResources extends Command
{
protected $signature = 'import:resources {csv_file_name}';
protected $description = 'Import resources into the API via a CSV file.';
protected $csv_file_name = null;
protected $csv_file_pathinfo = [];
protected $csv_headers = null;
protected $csv_data = [];
protected static $required_headers = [
'resource groups',
'category',
'title'
];
protected static $cached_langages = [];
protected static $cached_categories = [];
protected static $language_slug_replacement = [
'uk' => 'en'
];
public function handle()
{
ini_set('auto_detect_line_endings', true);
$this->csv_file_name = $this->argument('csv_file_name');
try {
// exception will be thrown if file is invalid
$this->csv_file_pathinfo = $this->checkFileProperties($this->csv_file_name);
} catch (\Exception $exception) {
$this->output->writeln("<error>Error encountered:</error> {$exception->getMessage()}");
return false;
}
list($this->csv_headers, $this->csv_data) = $this->getFileData($this->csv_file_name);
$this->checkHeaders($this->csv_headers); // throws exception if fails
// we now have the resources list
foreach ($this->csv_data as $csv_row_number => $_resource) {
$language = null;
$resource_link = null;
// add here cases for the different kinds of resources we are importing
if (!empty($_resource['uk asset'])) {
$language = $this->getLanguageFor('uk');
$resource_link = $_resource['uk asset'];
} else if (!empty($_resource['fr asset'])) {
$language = $this->getLanguageFor('fr');
$resource_link = $_resource['fr asset'];
} else if (!empty($_resource['de asset'])) {
$language = $this->getLanguageFor('de');
$resource_link = $_resource['de asset'];
} else if (!empty($_resource['no asset'])) {
$language = $this->getLanguageFor('no');
$resource_link = $_resource['no asset'];
}
// check for duplicates
if ($this->isDuplicateResource($language->id, $resource_link)) {
$this->output->writeln("<comment>Skipped row due to duplicate:</comment> {$csv_row_number}");
continue;
}
$resource = new Resource();
$resource->language_id = $language->id;
$resource->name = $_resource['title'];
$resource->link = $resource_link;
if ($resource->save() !== true) {
$this->output->writeln("<error>Failed to save row:</error> {$csv_row_number}");
}
// attach it to the category (this is a belongsToMany relationship. A resource can be in multiple categories, intended for admin)
$resource->categories()->attach($this->getResourceCategory($_resource['category'], $_resource['resource groups']));
}
}
protected function getResourceCategory($category, $parent_category)
{
if (isset(static::$cached_categories[$category])) {
return static::$cached_categories[$category];
}
$resource_category = ResourceCategory::firstOrCreate([
'name' => $category,
'parent_category' => $parent_category
]);
if (!$resource_category->exists()) {
$resource_category->save();
}
static::$cached_categories[$category] = $resource_category;
return $resource_category;
}
protected function isDuplicateResource($language_id, $resource_link)
{
return Resource::where('link', $resource_link)->where('language_id', $language_id)->count();
}
protected function getLanguageFor($language_slug)
{
// convert uk to en just incase it comes up
if (in_array($language_slug, array_keys(static::$language_slug_replacement))) {
$language_slug = static::$language_slug_replacement[$language_slug];
}
if (isset(static::$cached_langages[$language_slug])) {
return static::$cached_langages[$language_slug];
}
$language = Language::where('slug', $language_slug)->firstOrFail();
static::$cached_langages[$language_slug] = $language;
return $language;
}
protected function checkHeaders($csv_headers)
{
foreach (static::$required_headers as $required_header) {
if (in_array($required_header, $csv_headers) === false) {
throw new \Exception ("Missing required header in CSV: {$required_header}");
}
}
}
protected function getFileData($csv_file_name)
{
$file_handle = fopen($csv_file_name, 'r');
$csv_headers = null;
$csv_data = [];
$row_number = 0;
$empty_rows = 0; // if 3 empty rows are encountered we just break the loop- assumed end of data
while (($csv_row = fgetcsv($file_handle)) !== false) {
$row_number++; // the header is also a row in the CSV
if ($csv_headers === null) {
$_csv_headers = [];
foreach ($csv_row as $header) {
$_csv_headers[] = trim(strtolower($header));
}
$csv_headers = $_csv_headers;
unset($_csv_headers);
continue;
}
// skip empty rows
if (array_filter(array_unique($csv_row)) === []) {
$empty_rows++;
if ($empty_rows >= 3) {
// assume we have reached the end of the data
$this->output->writeln("<comment>Assumed end of data:</comment> {$row_number}");
break;
} else {
$this->output->writeln("<comment>Skip empty row:</comment> {$row_number}");
continue;
}
}
$_csv_data = [];
foreach ($csv_headers as $header_index => $header) {
$_csv_data[$header] = trim($csv_row[$header_index]); // we trim to ensure no additional spaces
// ensure the data is expected - primarily for the URL's
$validation_method = camel_case('validate ' . $header);
if (method_exists($this, $validation_method) && !empty($_csv_data[$header])) {
if (!$this->$validation_method($_csv_data[$header])) {
throw new \Exception("Issue with {$header} on row: {$row_number}");
}
}
}
$csv_data[$row_number] = $_csv_data; // add the row to the data with same row number as that in the CSV
unset($_csv_data);
}
return [$csv_headers, $csv_data];
}
protected function checkFileProperties($csv_file_name)
{
if (file_exists($csv_file_name) === false) {
throw new \Exception('File does not exist');
}
$csv_file_pathinfo = pathinfo($csv_file_name);
// extension is only defined if one is provided, eg /etc/hosts will have no extension
if (!isset($csv_file_pathinfo['extension']) || $csv_file_pathinfo['extension'] !== 'csv') {
throw new \Exception('File is not CSV');
}
return $csv_file_pathinfo;
}
protected function validateUkAsset($uk_asset)
{
return $this->validateUrl($uk_asset);
}
protected function validateFrAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateDeAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateNoAsset($asset)
{
return $this->validateUrl($asset);
}
protected function validateUrl($asset)
{
return filter_var($asset, FILTER_VALIDATE_URL);
}
}
\ No newline at end of file
......@@ -2,6 +2,7 @@
namespace App\Console;
use App\Console\Commands\ImportResources;
use Illuminate\Console\Scheduling\Schedule;
use Laravel\Lumen\Console\Kernel as ConsoleKernel;
......@@ -13,7 +14,7 @@ class Kernel extends ConsoleKernel
* @var array
*/
protected $commands = [
//
ImportResources::class
];
/**
......
......@@ -6,6 +6,11 @@ use Illuminate\Database\Eloquent\Model;
class ResourceCategory extends Model
{
protected $fillable = [
'name',
'parent_category'
];
public function resources()
{
return $this->hasMany(\App\Resource::class);
......
......@@ -122,34 +122,6 @@ class Initial extends Migration
],
];
static $resource_categories = [
["Accounts", "banking"],
["Bonds", "investments"],
["Borrowing money", "loans"],
["Budgeting", "budgeting"],
["Car repayments", "loans"],
["Cash", "banking"],
["Credit cards", "loans"],
["Credit options", "loans"],
["Debit cards", "banking"],
["Interest", "loans"],
["Interest rates", "loans"],
["Loans", "loans"],
["Payday loans", "loans"],
["Phone deals", "loans"],
["Phone Depreciation", "investments"],
["Saving for a car", "banking"],
["Savings", "banking"],
["Startups", "investments"],
["Store credit", "loans"],
["Prepaid Cards", "banking"],
["Packaged Accounts", "banking"],
["Mortgages", "loans"],
["Retirement", "investments"]
];
static $resources = '/resources/data/resources.csv';
protected $created_at = null;
protected $language = null;
......@@ -397,38 +369,6 @@ class Initial extends Migration
$blueprint->timestamps();
}
);
foreach (static::$resource_categories as $_resource_category) {
list($name, $parent_category) = $_resource_category;
$resource_category = new \App\ResourceCategory;
$resource_category->name = $name;
$resource_category->parent_category = $parent_category;
$resource_category->save();
}
$resources_csv_handle = fopen(APP_ROOT . static::$resources, 'r');
$headers = false;
while (($_resource = fgetcsv($resources_csv_handle)) !== false) {
if (empty($headers)) {
$headers = true;
continue;
}
$category = \App\ResourceCategory::where('name', 'LIKE', $_resource[1])->first();
if (!$category) {
exit ('category missing:' . $_resource[1]);
}
$resource = new \App\Resource;
$resource->language_id = $this->language->id;
$resource->name = $_resource[2];
$resource->link = $_resource[3];
$resource->save();
$resource->categories()->syncWithoutDetaching($category);
}
}
/**
......
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<title>Test questions</title>
<link href="https://fonts.googleapis.com/css?family=Roboto" rel="stylesheet">
<style>
* {
box-sizing: border-box;
}
html, body {
margin: 0;
padding: 0;
}
html {
height: 100%;
width: 100%;
}
body {
font-family: 'Roboto', 'Oxygen', 'Open Sans', 'Arial', sans;
font-size: 16px;
display: flex;
align-items: center;
justify-content: center;
flex-wrap: wrap;
height: 100%;
width: 100%;
background: rgb(220, 220, 220);
}
.view {
width: 360px;
height: 640px;
background: white;
margin: 30px;
padding: 30px;
}
ul, ol {
list-style: none;
margin: 0;
padding: 0;
}
hr {
border-top: 0;
border-bottom: 1px solid rgb(235, 235, 235);
margin: 32px 0;
}
.question, .answer-list {
display: flex;
flex-direction: column;
align-items: center;
}
.answer-list {
padding-bottom: 16px;
border-bottom: 1px solid rgb(235, 235, 235);
margin-bottom: 32px;
}
.budget-list {
border-bottom: 1px solid rgb(235, 235, 235);
padding-bottom: 8px;
margin-bottom: 8px;
}
.budget-title {
margin: 0 0 8px;
}
.budget-list li {
padding: 6px 0;
display: flex;
flex-direction: row;
justify-content: space-between;
}
.budget-list li strong {
flex: 1 1 auto;
font-weight: normal;
}
.budget-list li em {
flex: 0 0 auto;
font-style: normal;
}
.budget-list li.positive {
color: green;
}
.budget-list li.negative {
color: red;
}
.budget-list li.neutral {
color: orange;
}
.question-title {
font-size: 22px;
display: block;
text-align: center;
padding: 18px 0 24px;
border-bottom: 1px solid rgb(235, 235, 235);
margin-bottom: 32px;
width: 300px;
}
.answer {
display: inline-block;
border-radius: 3px;
padding: 8px 12px;
background: rgb(230, 230, 230);
text-align: center;
width: 300px;
cursor: pointer;
margin-bottom: 16px;
}
.answer.selected {
background: purple;
color: white;
}
.refresh {
display: inline-block;
border-radius: 3px;
padding: 8px 12px;
background: rgb(230, 230, 230);
text-align: center;
width: 100%;
max-width: 300px;
cursor: pointer;
margin-bottom: -100px; /* just so it doesnt affect the layout of the income title */
position: relative;
top: -48px;
}
.continue {
display: inline-block;
border-radius: 3px;
padding: 8px 12px;
background: rgb(230, 230, 230);
text-align: center;
width: 100%;
max-width: 300px;
cursor: pointer;
margin-bottom: 16px;
}
.continue.disabled {
color: rgb(120, 120, 120);
cursor: not-allowed;
}
</style>
<script>
const TestQuestions = <?= $questions; ?>;
const BudgetItems = <?= $budget_items; ?>;
window.player = {
language: 'en',
progress: 0,
character: {
name: 'Jack'
}
};
window.modifiers = {
commitedModifiers: [],
temporaryModifiers: [],
push(modifiers) {
// https://stackoverflow.com/questions/18884249/checking-whether-something-is-iterable
function isIterable(obj) {
// checks for null and undefined
if (obj == null) {
return false;
}
return typeof obj[Symbol.iterator] === 'function';
}
if (isIterable(modifiers)) {
for (var modifier of modifiers) {
this.temporaryModifiers.push(modifier);
}
} else {
this.temporaryModifiers.push(modifiers);
}
},
commit() {
for (var modifier of this.temporaryModifiers) {
this.commitedModifiers.push(modifier);
}
this.destroy();
},
destroy() {
this.temporaryModifiers = [];
},
get() {
// clone it with a temporary stamp
var _temporaryModifiers = [];
for (var modifier of this.temporaryModifiers) {
var _modifier = JSON.parse(JSON.stringify(modifier));
_modifier.temporary = true;
_temporaryModifiers.push(_modifier);
}
return this.commitedModifiers.concat(_temporaryModifiers);
},
get totals() {
var categories = this.calculate();
var totals = {
savings_item: null,
income_item: null,
outgoing_item: null
};
var defer = [];
var _categories = Object.keys(categories);
for (var i = 0; i < _categories.length; i++) {
var item = categories[_categories[i]];
switch (item.income_outgoing_total) {
case 'income':
case 'outgoing':
defer.push(item);
break;
case 'total':
totals[item.category + '_item'] = item;
break;
}
}
for (var item of defer) {
totals[item.income_outgoing_total + '_item'].amount += item.amount;
}
return totals;
},
calculate() {
var categories = {};
for (var modifier of modifiers.get()) {
if (!categories[modifier.category]) {
// if we don't do a shallow clone then the modifier will be doubled every time we recalculate
// due to pointers in JS.
categories[modifier.category] = JSON.parse(JSON.stringify(modifier));
} else {
var change = null;