周海龙 / globalso · 提交

转到一个项目

作者 zhl
提交 285a89dbc8cc77f099c4f8feb4d4f2b71eb08ab5 1 个父辈 b5e3764d

拆分同步脚本

内嵌 并排对比
正在显示 1 个修改的文件 包含 33 行增加 和 80 行删除
... ... @@ -19,6 +19,7 @@ use App\Services\ProjectServer;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Redis;
/**
* Class SyncInquiryProject
... ... @@ -38,7 +39,7 @@ class SyncInquiryProject extends Command
*
* @var string
*/
protected $description = '同步询盘信息：项目以及对应路由，';
protected $description = '同步询盘信息：项目';
/**
* Create a new command instance.
... ... @@ -61,8 +62,6 @@ class SyncInquiryProject extends Command
$this->syncGloV5();
echo '开始同步v6' . PHP_EOL;
$this->syncGloV6();
echo '删除过期数据' . PHP_EOL;
$this->deleteExpire();
return true;
}
... ... @@ -72,73 +71,47 @@ class SyncInquiryProject extends Command
*/
public function syncGloV5()
{
$date = intval(date('Ymd'));
$result = http_get('https://www.quanqiusou.cn/extend_api/webs/globalso_v5_tg.php');
$channel = Channel::pluck('contact_mobile', 'id')->toArray();
foreach ($result as $key=>$val)
{
echo '同步项目：' . $val['postid'] . ' - ' . $val['company'] . PHP_EOL;
// 记录渠道商, 如果渠道有问题， 记录日志， 跳过当前项目， 处理数据后， 第二天再重新同步
$channel_flag = false;
$channel_tmp = [];
foreach ($val['agent'] as $v) {
$channel_id = array_search($v, $channel);
// 代理信息 只要找到一个， 就需要继续记录
if (empty($channel_id)) {
$this->log('ERROR 代理商找不到不记录当前项目, 代理商手机号码：' . $v);
$channel_flag = true;
break;
continue;
}
array_push($channel_tmp, $channel_id);
}
if ($channel_flag || empty($channel_tmp)) {
if (empty($channel_tmp)) {
$this->log('ERROR 代理商找不到不记录当前项目, 项目ID：' . $val['postid']);
continue;
}
// 保存项目 以及路由
$project = InquiryProject::saveProject($val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
// 保存项目 以及 获取路由信息
$project = InquiryProject::saveProject($date, $val['v'], $val['postid'], $val['company'], implode(',', $channel_tmp), $val['main_url'], $val['is_split'], $val['test_url']);
$route_domain = $val['is_split'] && $val['test_url'] ? $val['test_url'] : $val['main_url'];
$this->syncGloV5Route($project->id, $route_domain);
}
return true;
}
/**
* 同步v4 v5项目路由
* @param $project_id
* @param $route_domain
* @return bool
*/
public function syncGloV5Route($project_id, $route_domain)
{
try{
$result = http_get($route_domain . 'k_u_api.php');
if (empty($result)) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
return false;
}
} catch (\Exception $e) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 未获取到路由信息：' . $project_id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
return false;
}
foreach ($result as $key=>$val) {
try {
$tmp = explode('|', $val);
$url_tmp = parse_url($tmp[0]);
$route = trim($url_tmp['path'], '/');
$title = str_replace('+', ' ', $tmp[1]);
if (strlen($title) > 200 || strlen($route) > 200) {
$this->log('syncGloV5Route 路由或标题过长，无效记录');
continue;
try{
$result = http_get($route_domain . 'k_u_api.php');
if (empty($result)) {
$this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php');
return false;
}
InquiryProjectRoute::saveProjectRoute($project_id, $title, $route);
file_put_contents(storage_path('logs/sync_inquiry_project_route/' . $project->id . '.json'), json_encode($result));
$this->pushTask($project->id);
} catch (\Exception $e) {
$this->log('syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 解析路径：' . $val . ', 错误信息：' . $e->getMessage() . PHP_EOL;
$this->log('syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage());
echo 'syncGloV5Route 未获取到路由信息：' . $project->id . ', 路由获取地址：' . $route_domain . 'k_u_api.php' . ', 错误信息：' . $e->getMessage() . PHP_EOL;
return false;
}
}
$this->deleteExpire($date);
return true;
}
... ... @@ -148,6 +121,7 @@ class SyncInquiryProject extends Command
*/
public function syncGloV6()
{
$date = intval(date('Ymd'));
// 获取优化中台项目
$project = Project::leftJoin('gl_project_online_check', 'gl_project.id', '=', 'gl_project_online_check.project_id')
->where('gl_project.type', Project::TYPE_TWO)
... ... @@ -165,55 +139,34 @@ class SyncInquiryProject extends Command
// 过滤暂停优化项目, 映射类型2， model没有定义常量
if (in_array(2, $val->level))
continue;
$project = InquiryProject::saveProject(InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
$this->syncGloV6Route($project->id, $val->id);
$project = InquiryProject::saveProject($date,InquiryProject::VERSION_SIX, $val->id, $val->title, $val->channel['channel_id'], $domains[$val->id]);
$this->pushTask($project->id);
}
$this->deleteExpire($date);
return true;
}
/**
* 同步v6项目路由
* @param $project_id
* @param $origin_project_id
* 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
* @param $date
* @return bool
*/
public function syncGloV6Route($project_id, $origin_project_id)
public function deleteExpire($date)
{
ProjectServer::useProject($origin_project_id);
// TODO 产品分类标题、路由， 产品标题、路由， 同步到路由表
$category = Category::where('status', Category::STATUS_ACTIVE)->get(['title', 'route']);
foreach ($category as $key=>$val) {
InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
}
// 产品数量会比较多， 所以使用分页 同步数据
$id = 0;
while (true) {
echo '同步项目路由：' . $id . PHP_EOL;
$product = Product::where('status', Product::STATUS_ON)->where('id', '>', $id)->orderBy('id', 'asc')->limit(1000)->get(['id', 'title', 'route']);
if ($product->isEmpty())
break;
foreach ($product as $key=>$val) {
$id = $val->id;
InquiryProjectRoute::saveProjectRoute($project_id, $val->title, $val->route);
}
}
DB::disconnect('custom_mysql');
$project_num = InquiryProject::where('date', '<', $date)->delete();
$this->log('删除过期项目数量：' . $project_num);
return true;
}
/**
* 删除过期数据， 非当日更新数据， 都删除， 误删第二天再重新同步更新
* 往任务队列中生产数据
* @param $id
* @return bool
*/
public function deleteExpire()
public function pushTask($id)
{
$date = intval(date('Ymd'));
$project_num = InquiryProject::where('date', '<', $date)->delete();
$project_route_num = InquiryProjectRoute::where('date', '<', $date)->delete();
$this->log('删除过期项目数量：' . $project_num . ', 删除过期路由数量：' . $project_route_num);
Redis::lPush('sync_inquiry_project_route_task', $id);
return true;
}
... ...