长春做商业平台网站,网站建设岗位的任职资格,个人网站建立多少钱,汉阴网站建设一、需求#xff1a;错误日志Top10告警发送
二、需求分解
jsoup实现登录#xff0c;获取到cookie和token等用户鉴权信息获取接口相应的key值调用日志平台错误日志Top榜接口#xff0c;查询到结果集调用企业微信机器人发送消息接口加上定时任务#xff0c;可以实现定时发送…一、需求错误日志Top10告警发送
二、需求分解
jsoup实现登录获取到cookie和token等用户鉴权信息获取接口相应的key值调用日志平台错误日志Top榜接口查询到结果集调用企业微信机器人发送消息接口加上定时任务可以实现定时发送错误日志告警的功能后续加进定时任务里面去
jsoup是java的爬虫框架可以爬取网页数据这里没有重点使用只是做了个登录功能。后续可以专门它写一份爬虫的程序。。 dependencygroupIdorg.jsoup/groupIdartifactIdjsoup/artifactIdversion1.12.1/version/dependency
package com.smy.cbs.task;import cn.hutool.core.date.DateUtil;
import cn.hutool.core.util.StrUtil;
import cn.hutool.http.HttpRequest;
import cn.hutool.http.HttpResponse;
import cn.hutool.http.HttpUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import com.smy.cbs.util.RetryUtil;
import com.smy.framework.core.support.SpringTestCase;
import org.apache.commons.collections4.CollectionUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.junit.Test;
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import javax.annotation.Resource;
import javax.net.ssl.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;/*** author youlu* ClassName HttpTestt* Date 2023/11/22 17:14* Version V1.0**/
public class HttpTestt2 extends SpringTestCase {Resourceprivate RetryUtil retryUtil;public static final String LOG_CENTER_BASE_URL https://logxxxxx.com/api/v1/;public static final String USER_NAME aaaaa;public static final String PASSWORD bbbbb;public static String X_Csrf_Token ;public static String COOKIE ;public static String id ;public static int maxCount Integer.MAX_VALUE;//达到阈值则告警public static ListString filterContent Lists.newArrayList();//过滤内容public static int SUB_CONTENT_LENGTH 240;//截取报错内容字符串长度public static int SUB_TITLE_LENGTH 20;//截取报错类型字符串长度public static int PERIOD_HOUR 24*7;//24小时内的错误日志public static ListString SEARCH_SYSTEM Lists.newArrayList(cbs_core, adv_core, rls_core, uts_core);//查询的系统//public static ListString SEARCH_SYSTEM Lists.newArrayList(adv_core);//查询的系统public static String WX_ROBOT_URL https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key05e1f4a8-aaaaaa /* ,https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key87140cb7-bbbbbb*/;public static final String TEMPLATE {\markdown\:{\content\:\%s\},\msgtype\:\markdown\};public static final String QUERY_TEMPLATE {\n \app\: \system\,\n \source\: \other\,\n \query\: \repo\\\smy_%s\\\ origin\\\*\\\ AND \\\ERROR\\\\\n| where level\\\ERROR\\\\\n| eval err_typearr_index(split(arr_index(split(_raw, \\\ - \\\), 1), \\\\\\\d\\\), 0)\\n| stats count() as num by err_type\\n| sort 10 by num\\n| join typeinner err_type [\\n repo\\\smy_%s\\\ origin\\\*\\\ AND \\\ERROR\\\\\n | where level\\\ERROR\\\\\n | eval err_typearr_index(split(arr_index(split(_raw, \\\ - \\\), 1), \\\\\\\d\\\), 0)\\n | fields err_type, _raw\\n | dedup err_type\\n]\\n| rename _raw as 原始日志, num as 统计, err_type as 错误类型\,\n \mode\: \smart\,\n \preview\: false,\n \collectSize\: -1,\n \timeout\: 1000,\n \sorts\: []\n };public static ThreadPoolExecutor threadPool new ThreadPoolExecutor(10, 10, 0, TimeUnit.MINUTES,new LinkedBlockingQueueRunnable(100), new CustomizableThreadFactory(Log_Center-pool-));Testpublic void LogCenterData() throws IOException {//1.模拟登录 jsoupjsoupLogin();for (String systemName : SEARCH_SYSTEM) {try {Thread.sleep(1000);//2.获取keyString id getKey(systemName);Thread.sleep(8000);ListMapString, Object logCenterList retryUtil.doRetry(4, () - {//3.获取查询结果ListMapString, Object contentList getContentList(systemName, id);if (CollectionUtils.isEmpty(contentList)) {Thread.sleep(8000);throw new Exception(systemName 未查询到数据需要重试);}return contentList;}, systemName 获取日志数据);//4.调微信发送短信//if (adv_core.equals(systemName) CollectionUtils.isNotEmpty(logCenterList)) {// ListListMapString, Object partition Lists.partition(logCenterList, 5);// partition.stream().forEach(k - sendWxMessage(systemName, id, k));//} else {sendWxMessage(systemName, id, logCenterList);//}} catch (Exception e) {e.printStackTrace();}}}public static String getKey(String systemName) {Date endDate new Date();Date startDate DateUtil.offsetHour(endDate, -PERIOD_HOUR);//请求参数JSONObject paramJson JSON.parseObject(String.format(QUERY_TEMPLATE, systemName, systemName));paramJson.put(startTime,startDate.getTime());paramJson.put(endTime,endDate.getTime());HttpResponse execute HttpRequest.post(LOG_CENTER_BASE_URL jobs)//设置请求头(可任意加).header(X-Csrf-Token, X_Csrf_Token).header(Cookie, COOKIE).header(Content-Type, application/json).header(Connection,keep-alive)//请求参数.body(paramJson.toJSONString()).timeout(40000).execute();String body1 execute.body();String id JSON.parseObject(body1).getString(id);return id;}public static ListMapString,Object getContentList(String systemName,String id) {String url LOG_CENTER_BASE_URL jobs/ id /results;url ?_ System.currentTimeMillis();HttpResponse execute HttpRequest.get(url)//设置请求头(可任意加).header(X-Csrf-Token, X_Csrf_Token).header(Cookie, COOKIE).header(Content-Type, application/json).header(User-Agent, Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36).header(Sec-Ch-Ua-Platform, Windows).header(Connection,keep-alive).timeout(40000).execute();JSONArray rows JSON.parseObject(execute.body()).getJSONArray(rows);System.err.println(rows.toString());ListMapString, Object list Lists.newArrayList();for (Object row : rows) {try {JSONArray jsonArray JSON.parseArray(row.toString());String title jsonArray.get(0).toString();Integer count Integer.valueOf(jsonArray.get(1).toString());String content jsonArray.get(2).toString();if (count maxCount) {continue;}if (filterContent.stream().anyMatch(k - content.contains(k))) {continue;}int subContentLen content.length() SUB_CONTENT_LENGTH ? content.length() : SUB_CONTENT_LENGTH;String subContent content.substring(0, subContentLen);int subTitleLen title.length() SUB_TITLE_LENGTH ? title.length() : SUB_TITLE_LENGTH;String subTitle content.substring(0, subTitleLen);MapString, Object map new HashMap();map.put(title, subTitle);map.put(count, count);map.put(content, subContent);list.add(map);} catch (Exception e) {e.printStackTrace();}}if (CollectionUtils.isEmpty(list)) {System.err.println(systemName :未获取到数据请求链接 url token: X_Csrf_Token cookie: COOKIE);}return list;}public static void sendWxMessage(String systemName, String id, ListMapString, Object contentList) {if (CollectionUtils.isEmpty(contentList)) {System.err.println(systemName 发送内容为空不发送机器人微信消息,id: id);return;}String periodDesc getPeriodDesc();final int[] topNum {1};String StringContent contentList.stream().map(k - {String title (String) k.get(title);Integer count (Integer) k.get(count);String content (String) k.get(content);//return String.format( top-%d%s\n 出现次数%d\n 错误详情描述: %s, topNum[0], title, count, content.trim());return String.format( top-%d出现次数%d\n 错误详情描述: %s, topNum[0], count, content.trim());}).collect(Collectors.joining(\n\n));String content String.format(%s近%s内错误日志Top10\n%s, systemName, periodDesc, StringContent);String sendContent String.format(TEMPLATE, content);////MapString, String contentMap new HashMap();//contentMap.put(content, content);//MapString, String markdownMap new HashMap();//markdownMap.put(markdown, JSON.toJSONString(contentMap));//markdownMap.put(msgtype, markdown);//String sendText JSON.toJSONString(markdownMap);////System.err.println(sendText);Arrays.stream(WX_ROBOT_URL.split(,)).forEach(wx - {//String post HttpUtil.post(wx debug1, sendText);//String post2 HttpUtil.post(wx debug1, sendContent);String post2 HttpUtil.post(wx, sendContent);System.err.println(systemName 发送微信情况2 post2);//System.err.println(systemName 发送微信情况 post);});}private static String getPeriodDesc() {if (PERIOD_HOUR 24) {return PERIOD_HOUR 小时;}BigDecimal dayBigDecimal BigDecimal.valueOf(PERIOD_HOUR).divide(BigDecimal.valueOf(24), 2, BigDecimal.ROUND_HALF_UP);String s StrUtil.removeSuffix(StrUtil.removeSuffix(String.valueOf(dayBigDecimal), 00), 0);String[] split s.split(\\.);if (split.length 1) {return split[0] 天;}return s 天;}/*** Jsoup 模拟登录 访问个人中心* 先构造登录请求参数成功后获取到cookies* 设置request cookies再次请求* throws IOException*/public static void jsoupLogin() throws IOException {//Jsoup加这个避免请求https报证书问题trustEveryone();// 构造登陆参数MapString,String data new HashMap();data.put(username, USER_NAME);data.put(password, PASSWORD);Connection.Response response Jsoup.connect(LOG_CENTER_BASE_URL account/ldap/login).ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).followRedirects(false) // 禁止重定向.postDataCharset(utf-8).header(Upgrade-Insecure-Requests,1).header(Accept,application/json).header(Content-Type,application/json).header(User-Agent,Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36).requestBody(JSON.toJSONString(data)).method(Connection.Method.POST).execute();response.charset(UTF-8);// login 中已经获取到登录成功之后的cookies// 构造访问个人中心的请求MapString, String cookies response.cookies();cookies.forEach((k, v) - COOKIE k v);String body response.body();X_Csrf_Token JSON.parseObject(body).getString(X-Csrf-Token);System.err.println(COOKIE: COOKIE);System.err.println(X_Csrf_Token: X_Csrf_Token);}public static void jsoupHandle(String id) throws IOException {String url LOG_CENTER_BASE_URL jobs/ id /results;Document document Jsoup.connect(url).header(X-Csrf-Token, X_Csrf_Token).header(Cookie, COOKIE).ignoreContentType(true) // 忽略类型验证.ignoreHttpErrors(true).method(Connection.Method.GET).get();System.err.println(url);String s JSON.toJSONString(document);System.err.println(s);}public static void trustEveryone() {try {HttpsURLConnection.setDefaultHostnameVerifier(new HostnameVerifier() {public boolean verify(String hostname, SSLSession session) {return true;}});SSLContext context SSLContext.getInstance(TLS);context.init(null, new X509TrustManager[] { new X509TrustManager() {public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {}public X509Certificate[] getAcceptedIssuers() {return new X509Certificate[0];}} }, new SecureRandom());HttpsURLConnection.setDefaultSSLSocketFactory(context.getSocketFactory());} catch (Exception e) {// e.printStackTrace();}}}
三、实现效果 企业微信机器狗开发者文档群机器人配置说明 - 接口文档 - 企业微信开发者中心
