作者 lyh

Merge branch 'master' of http://47.244.231.31:8099/zhl/globalso-v6 into develop

... ... @@ -153,9 +153,12 @@ class HtmlCollect extends Command
case 'blog':
$source = RouteMap::SOURCE_BLOG;
break;
default:
case 'post':
$source = RouteMap::SOURCE_PRODUCT;
break;
default:
$source = RouteMap::SOURCE_MODULE;
break;
}
$complete = false;
... ... @@ -260,6 +263,7 @@ class HtmlCollect extends Command
//判断资源是否需要下载
protected function url_check($url, $project_id, $domain, $web_url_domain, $home_url)
{
$url = trim($url);
if ($url) {
$url = str_replace('"', '', $url);
$arr = parse_url($url);
... ...
... ... @@ -155,9 +155,12 @@ class HtmlLanguageCollect extends Command
case 'blog':
$source = RouteMap::SOURCE_BLOG;
break;
default:
case 'post':
$source = RouteMap::SOURCE_PRODUCT;
break;
default:
$source = RouteMap::SOURCE_MODULE;
break;
}
$complete = false;
... ... @@ -276,6 +279,7 @@ class HtmlLanguageCollect extends Command
//判断资源是否需要下载
protected function url_check($url, $project_id, $domain, $web_url_domain, $home_url)
{
$url = trim($url);
if ($url) {
$url = str_replace('"', '', $url);
$arr = parse_url($url);
... ...
... ... @@ -10,6 +10,9 @@ use App\Models\Collect\CollectSource;
use App\Models\Collect\CollectTask;
use App\Models\Com\UpdateLog;
use App\Models\Com\UpdateOldInfo;
use App\Models\CustomModule\CustomModule;
use App\Models\CustomModule\CustomModuleCategory;
use App\Models\CustomModule\CustomModuleContent;
use App\Models\News\News;
use App\Models\News\NewsCategory;
use App\Models\Product\Category;
... ... @@ -187,6 +190,7 @@ class ProjectUpdate extends Command
if (isset($data['code']) && $data['code'] == 200) {
$phones = $data['data']['phones'] ?? '';
$emails = $data['data']['emails'] ?? '';
$custom_types = $data['data']['customposttypes'] ?? '';
$model = new WebSettingReceiving();
if ($phones) {
... ... @@ -229,6 +233,25 @@ class ProjectUpdate extends Command
}
}
}
if ($custom_types) {
$custom_model = new CustomModule();
foreach ($custom_types as $v_custom) {
try {
$custom_info = $custom_model->read(['route' => $v_custom]);
if (!$custom_info) {
$custom_model->add([
'name' => $v_custom,
'project_id' => $project_id,
'route' => $v_custom
]);
}
UpdateLog::createLog($project_id, $v_custom, $task->api_url);
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
continue;
}
}
}
} else {
return true;
}
... ... @@ -392,7 +415,7 @@ class ProjectUpdate extends Command
$category_id = $logic->getCategory(array_column($category_arr, 'id'));
}
//图片
if (is_array($item['images'])) {
if (is_array($item['images'] ?? '')) {
$image = $item['images'][0] ?? '';
} else {
$image = $item['images'] ?? '';
... ... @@ -448,7 +471,7 @@ class ProjectUpdate extends Command
} else {
return true;
}
} else {
} elseif ($api_type == 'page') {
//单页
$url = $api_url . '?' . http_build_query(['w' => 'page', 'page' => 1, 'pagesize' => 0]);
$data = curl_c($url);
... ... @@ -510,6 +533,79 @@ class ProjectUpdate extends Command
} else {
return true;
}
} else {
//自定义模块
$custom_model = new CustomModule();
$custom_info = $custom_model->read(['route' => $api_type]);
if ($custom_info) {
$url = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => 1, 'pagesize' => 0]);
$data = curl_c($url);
if (isset($data['code']) && $data['code'] == 200) {
$category = $data['data']['category'] ?? [];
$this->category_custom_insert($project_id, $custom_info['id'], $category, 0);
$count = $data['data']['count'] ?? 0;
$total_page = ceil($count / $page_size);
for ($page = 1; $page <= $total_page; $page++) {
$url_page = $api_url . '?' . http_build_query(['w' => $api_type, 'page' => $page, 'pagesize' => $page_size]);
$data_page = curl_c($url_page);
if (isset($data_page['code']) && $data_page['code'] == 200) {
$items = $data_page['data']['data'] ?? [];
$model = new CustomModuleContent();
$category_model = new CustomModuleCategory();
foreach ($items as $item) {
$route = $this->get_url_route($item['url'] ?? '');
if ($route) {
//分类
$category_id = '';
if ($item['category'] ?? []) {
$category_arr = $category_model->list(['original_id' => ['in', array_column($item['category'], 'id')]]);
$category_id = implode(',', array_column($category_arr, 'id'));
}
//名称去掉特殊符号
$item['title'] = $this->special2str($item['title'] ?? '');
try {
$custom_content = $model->read(['route' => $route], 'id');
if (!$custom_content) {
$id = $model->insertGetId([
'project_id' => $project_id,
'name' => $item['title'],
'module_id' => $custom_info['id'],
'category_id' => $category_id,
'content' => $item['content'] ?? '',
'created_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
'updated_at' => $item['post_time'] ?? date('Y-m-d H:i:s'),
'is_upgrade' => 1,
'six_read' => 1,
'route' => $route
]);
$this->set_map($route, RouteMap::SOURCE_MODULE, $id, $project_id);
CollectTask::_insert($item['url'], $project_id, RouteMap::SOURCE_MODULE, $id, $domain_arr['host'], $link_type, $language_list, $page_list);
} else {
$id = $custom_content['id'];
$model->edit([
'name' => $item['title'],
'category_id' => $category_id,
'content' => $item['content'] ?? '',
], ['id' => $id]);
}
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', task_id: ' . $task->id . ', error: ' . $e->getMessage() . PHP_EOL;
continue;
}
}
}
}
}
} else {
return true;
}
}
}
}
//关闭数据库
... ... @@ -595,7 +691,7 @@ class ProjectUpdate extends Command
$parent_id = $parent['id'];
}
if (!empty($item['children'])) {
if (!empty($item['children'] ?? [])) {
$this->category_insert($project_id, $item['children'], $parent_id);
}
}
... ... @@ -629,13 +725,48 @@ class ProjectUpdate extends Command
$parent_id = $parent['id'];
}
if (!empty($item['children'])) {
if (!empty($item['children'] ?? [])) {
$this->category_news_insert($project_id, $item['children'], $parent_id);
}
}
}
}
//扩展模块多级分类入库
protected function category_custom_insert($project_id, $module_id, $items, $pid = 0)
{
$model = new CustomModuleCategory();
foreach ($items as $item) {
$route = $this->get_url_route($item['url'] ?? '');
if ($route) {
$parent = $model->read(['pid' => $pid, 'route' => $route], 'id');
if (!$parent) {
try {
$item['name'] = $this->special2str($item['name'] ?? '');
$parent_id = $model->addReturnId([
'project_id' => $project_id,
'module_id' => $module_id,
'name' => $item['name'],
'pid' => $pid,
'original_id' => $item['id'],
'route' => $route
]);
$this->set_map($route, RouteMap::SOURCE_MODULE_CATE, $parent_id, $project_id);
} catch (\Exception $e) {
echo 'date:' . date('Y-m-d H:i:s') . ', category_custom_insert error: ' . $e->getMessage() . PHP_EOL;
continue;
}
} else {
$parent_id = $parent['id'];
}
if (!empty($item['children'] ?? [])) {
$this->category_custom_insert($project_id, $module_id, $item['children'], $parent_id);
}
}
}
}
//特殊字符转换
protected function special2str($str)
{
... ...
... ... @@ -3,10 +3,11 @@
namespace App\Console\Commands;
use App\Helper\Arr;
use App\Models\Domain\DomainInfo;
use App\Models\Project\DeployOptimize;
use App\Models\Product\Category;
use App\Models\Product\Product;
use App\Models\Project\OnlineCheck;
use App\Models\Project\Project;
use App\Models\RouteMap\RouteMap;
use App\Models\Template\BCustomTemplate;
use App\Services\ProjectServer;
use Carbon\Carbon;
use GuzzleHttp\Client;
... ... @@ -190,13 +191,13 @@ class WebTraffic extends Command
$data = [
'ip' => $ips[$project_key]['ip'],
'referrer_url' => $this->getReferer($ips[$project_key]['ip_area']),
'url' => $project['visit_urls'][$j],
'device_port' => $project['device_port'],
'domain' => $project['domain'],
'referrer_url' => $this->getReferer($ips[$project_key]['ip_area']),
'user_agent' => $project['user_agent'],
];
Log::channel('traffic')->info('traffic project_id:' . $project['project_id'], $data);
$promises[] = $client->postAsync($project['domain'] . 'api/customerVisit', ['form_params' => $data]);
$promises[] = $client->postAsync($project['domain'] . 'api/traffic_visit', ['form_params' => $data]);
}
if($promises){
... ... @@ -233,28 +234,29 @@ class WebTraffic extends Command
protected function getProjectList($type, $page){
//推广项目
$list = Project::with('domainInfo')
->leftJoin('gl_project_deploy_optimize', 'gl_project_deploy_optimize.project_id', '=', 'gl_project.id')
->where('gl_project_deploy_optimize.domain', '>', 0)
->leftJoin('gl_project_deploy_optimize as pdo', 'pdo.project_id', '=', 'gl_project.id')
->leftJoin('gl_project_online_check as poc', 'poc.project_id', '=', 'gl_project.id')
->where('pdo.domain', '>', 0)
->where('poc.qa_status', OnlineCheck::STATUS_ONLINE_TRUE)
->whereIn('gl_project.type', [Project::TYPE_TWO, Project::TYPE_FOUR])
->whereIn('gl_project_deploy_optimize.project_id', [6,25]) //todo 测试两个项目 后面删掉
->where(function ($query) use ($type){
->whereIn('pdo.project_id', [6, 25]) //todo 测试两个项目 后面删掉
->where(function ($query) use ($type) {
if($type == 1){
//1-3个月项目
$startTime = Carbon::now()->addMonths(-4)->toDateString();
$endTime = Carbon::now()->addMonths(-1)->toDateString();
$query->whereBetween('gl_project_deploy_optimize.start_date', [$startTime,$endTime]);
$query->whereBetween('pdo.start_date', [$startTime,$endTime]);
}elseif($type == 2){
//4-8个月项目
$startTime = Carbon::now()->addMonths(-9)->startOfDay()->toDateTimeString();
$endTime = Carbon::now()->addMonths(-4)->endOfDay()->toDateTimeString();
$query->whereBetween('gl_project_deploy_optimize.start_date', [$startTime,$endTime]);
$query->whereBetween('pdo.start_date', [$startTime,$endTime]);
}else{
//大于9个月项目
$startTime = Carbon::now()->addMonths(-9)->startOfDay()->toDateTimeString();
$query->whereBetween('gl_project_deploy_optimize.start_date', '<', $startTime);
$query->whereBetween('pdo.start_date', '<', $startTime);
}
})->select('gl_project_deploy_optimize.project_id')->forPage($page, 500)->get();
})->select('pdo.project_id')->forPage($page, 500)->get();
//其他地方在引流的域名
$other = DB::connection('projects_mysql')->table('projects')->where('switch', 1)->pluck('domain')->toArray();
... ... @@ -278,28 +280,17 @@ class WebTraffic extends Command
*/
protected function getProductUrls($project_id){
ProjectServer::useProject($project_id);
//产品分类页面
$product_cate_ids = DB::connection('custom_mysql')->table('gl_product_category')
->where('project_id', $project_id)->where('status', 1)->pluck('id')->toArray();
//只查发布的分类路由
$data['urls_cats'] = DB::connection('custom_mysql')->table('gl_route_map')
->where('project_id', $project_id)->where('source', RouteMap::SOURCE_PRODUCT_CATE)
->whereIn('source_id', $product_cate_ids)->get()->toArray();
//单页面
$page_ids = DB::connection('custom_mysql')->table('gl_web_custom_template')
->where('project_id', $project_id)->where('status', 1)->pluck('id')->toArray();
//只查发布的单页面
$data['urls_page'] = DB::connection('custom_mysql')->table('gl_route_map')
->where('project_id', $project_id)->where('source', RouteMap::SOURCE_PAGE)
->whereIn('source_id', $page_ids)->get()->toArray();
//产品详情页
$product_ids = DB::connection('custom_mysql')->table('gl_product')
->where('project_id', $project_id)->where('status', 1)->pluck('id')->toArray();
$data['urls_details'] = DB::connection('custom_mysql')->table('gl_route_map')
->where('project_id', $project_id)->where('source', RouteMap::SOURCE_PRODUCT)
->whereIn('source_id', $product_ids)->get()->toArray();
//已发布产品分类页面
$data['urls_cats'] = DB::connection('custom_mysql')->table('gl_product_category')
->where('project_id', $project_id)->where('status', Category::STATUS_ACTIVE)->pluck('route','id')->toArray();
//已发布单页面
$data['urls_page'] = DB::connection('custom_mysql')->table('gl_web_custom_template')
->where('project_id', $project_id)->where('status', BCustomTemplate::STATUS_ACTIVE)->pluck('url', 'id')->toArray();
//已发布产品详情页
$data['urls_details'] = DB::connection('custom_mysql')->table('gl_product')
->where('project_id', $project_id)->where('status', Product::STATUS_ON)->pluck('route', 'id')->toArray();
$data['urls_cats'] = array_merge($data['urls_cats'], $data['urls_page']);
if(empty($data['urls_cats'])){
... ... @@ -374,21 +365,24 @@ class WebTraffic extends Command
$url = [];
if($res_sdzb == 0){//深度一页
$url[] = $project_urls[$res_ymzb] ? Arr::random($project_urls[$res_ymzb])->route : '';
$url[] = $project_urls[$res_ymzb] ? Arr::random($project_urls[$res_ymzb]) : '';
}elseif($res_sdzb == 1){//深度两页
$url[] = $project_urls['home'];
$url[] = $project_urls[$res_ymzb] ? Arr::random($project_urls[$res_ymzb])->route : '';
$url[] = $project_urls[$res_ymzb] ? Arr::random($project_urls[$res_ymzb]) : '';
}elseif($res_sdzb == 2){//深度3-6页
$yms = rand(2,5); //随机页面数
$url = Arr::pluck(Arr::random($all_url, $yms), 'route');
$url = Arr::random($all_url, $yms);
$url = Arr::prepend($url, $project_urls['home']);//首页加到最前面去
}elseif($res_sdzb == 3){//深度7-11页
$yms = rand(6,10); //随机页面数
$url = Arr::pluck(Arr::random($all_url, $yms), 'route');
$url = Arr::random($all_url, $yms);
$url = Arr::prepend($url, $project_urls['home']);//首页加到最前面去
}
foreach ($url as &$v){
if(!Str::contains($v, $project_urls['home'])){
if (FALSE === strpos($v, '.htm')) {
$v .= '/';
}
$v = $project_urls['home'] . $v;
}
}
... ...
... ... @@ -26,9 +26,9 @@ class Kernel extends ConsoleKernel
$schedule->command('rank_data_week')->dailyAt('01:00')->withoutOverlapping(1); // 排名数据,每周一凌晨执行一次
// $schedule->command('share_user')->dailyAt('01:00')->withoutOverlapping(1); // 清除用户ayr_share数据,每天凌晨1点执行一次
$schedule->command('count')->dailyAt('01:00')->withoutOverlapping(1); //每天凌晨1点执行一次
// $schedule->command('web_traffic 1')->everyThirtyMinutes(); // 引流 1-3个月的项目,半小时一次
// $schedule->command('web_traffic 2')->cron('*/18 * * * *'); // 引流 4-8个月的项目,18分钟一次
// $schedule->command('web_traffic 3')->cron('*/12 * * * *'); // 引流 大于9个月的项目,12分钟一次
$schedule->command('web_traffic 1')->everyThirtyMinutes(); // 引流 1-3个月的项目,半小时一次
$schedule->command('web_traffic 2')->cron('*/18 * * * *'); // 引流 4-8个月的项目,18分钟一次
$schedule->command('web_traffic 3')->cron('*/12 * * * *'); // 引流 大于9个月的项目,12分钟一次
$schedule->command('sync_channel')->dailyAt('06:00')->withoutOverlapping(1); // 渠道信息,每天执行一次
$schedule->command('month_count')->monthlyOn(1,'01:00')->withoutOverlapping(1);//没月月初1号执行月统计记录
$schedule->command('forward_count')->monthlyOn(1,'01:00')->withoutOverlapping(1);//没月月初1号执行月统计转发询盘记录
... ...
... ... @@ -52,16 +52,8 @@ class Category extends Base
*/
public static function getProductNum($cate_id){
$cate_ids = self::getChildIdsArr($cate_id);
$str = 0;
foreach ($cate_ids as $v){
$info = self::where('pid',$v)->first();
if($info){
continue;
}else{
$count = Product::where('category_id','like','%,'.$v.',%')->count();
$str = $str+$count;
}
}
return $str;
$count = CategoryRelated::whereIn('cate_id',$cate_ids)->count();
// $str = 0;
return $count;
}
}
... ...
... ... @@ -11,4 +11,6 @@ class BCustomTemplate extends Base
protected $connection = 'custom_mysql';
const NOT_FOUND_PAGE_URL = '404';
const STATUS_ACTIVE = 1;
}
... ...