网站编程开发,眼镜网站建设,遵义做网站推广,新闻发稿平台有哪些?概述 基于Flume MongoDB#xff0c;对现有的多个应用系统进行日志采集。特点 采集范围每一次用户请求的请求信息。数据量大尽量减少现有系统的改动数据流图 说明#xff1a;首先考虑的结构体系#xff0c;是直接在应用系统中#xff0c;将日志数据写到Flume#xff1b;但…概述 基于Flume MongoDB对现有的多个应用系统进行日志采集。 特点 采集范围每一次用户请求的请求信息。数据量大尽量减少现有系统的改动数据流图 说明 首先考虑的结构体系是直接在应用系统中将日志数据写到Flume但是现有的应用系统都是非Maven的需要在每一个应用系统中添加20个jar包。为避免这种情况抽出了一层日志服务开放webservice服务给应用系统调用最终形成上述的体系。 日志存储 1.需要解决的问题 1.1 借助Flume写日志到MongoDB 参考Flume学习应用Java写日志数据到MongoDB - 外网参考Flume学习应用Java写日志数据到MongoDB 1.2 发布webservice服务 参考在web项目中发布jaxws - 外网参考在web项目中发布jaxws 2.日志服务实现 一个简单的web项目对外发布一个webservice服务实现写日志到Flume。 2.1 文件结构 src/main/java|---- cn.sinobest.asj.log|---- ISALog.java # 日志服务接口|---- SALogImpl.java # 日志服务实现类|---- cn.sinobest.asj.log.exception|---- InvalidGradeException.java # 表示无效的日志等级|---- InvalidFormatExceptioin.java # 表示无效的消息格式要求是JSON格式字符串|---- cn.sinobest.asj.log.util|---- ValidGrade.java # 枚举所有有效的日志等级DEBUG, INFO, WARN, ERROR|---- MessageTemplate.java # 消息模板
src/main/resources|---- log4j.properties
src/main/webapp|---- WEB-INF|---- sun-jaxws.xml|---- web.xml|---- index.jsp # 这个可以忽略
pom.xml 2.2 文件内容 你可以直接从log-service拿到源代码并跳过这一节的内容。 pom.xml 1 project xmlnshttp://maven.apache.org/POM/4.0.0 xmlns:xsihttp://www.w3.org/2001/XMLSchema-instance2 xsi:schemaLocationhttp://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd3 modelVersion4.0.0/modelVersion4 groupIdcn.sinobest.asj/groupId5 artifactIdlog-service/artifactId6 packagingwar/packaging7 version0.0.1-SNAPSHOT/version8 namelog-service Maven Webapp/name9 urlhttp://maven.apache.org/url
10 dependencies
11 dependency
12 groupIdjunit/groupId
13 artifactIdjunit/artifactId
14 version3.8.1/version
15 scopetest/scope
16 /dependency
17 dependency
18 groupIdlog4j/groupId
19 artifactIdlog4j/artifactId
20 version1.2.16/version
21 /dependency
22 dependency
23 groupIdcommons-logging/groupId
24 artifactIdcommons-logging/artifactId
25 version1.1.1/version
26 /dependency
27 !-- for log to Flume --
28 dependency
29 groupIdorg.apache.flume.flume-ng-clients/groupId
30 artifactIdflume-ng-log4jappender/artifactId
31 version1.6.0/version
32 /dependency
33 !-- for jax-ws --
34 dependency
35 groupIdcom.sun.xml.ws/groupId
36 artifactIdjaxws-rt/artifactId
37 version2.2.10/version
38 /dependency
39 !-- for test the log content is a json-format or not --
40 dependency
41 groupIdorg.mongodb/groupId
42 artifactIdmongo-java-driver/artifactId
43 version2.13.0/version
44 /dependency
45 /dependencies
46 build
47 finalNamelog-service/finalName
48 /build
49 /project pom.xml web.xml ?xml version1.0 encodingUTF-8?
web-app xmlns:xsihttp://www.w3.org/2001/XMLSchema-instancexmlnshttp://java.sun.com/xml/ns/javaeexmlns:webhttp://java.sun.com/xml/ns/javaee/web-app_3_0.xsdxsi:schemaLocationhttp://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsdidWebApp_ID version3.0 metadata-completefalsedisplay-nameArchetype Created Web Application/display-name
/web-app 注意如果是servlet3.0以下的版本需要额外的配置。 log4j.properties # 配置Log4jAppender能写日志到Flume
log4j.appender.flumeAvroorg.apache.flume.clients.log4jappender.Log4jAppender
log4j.appender.flumeAvro.Hostnamelocalhost
log4j.appender.flumeAvro.Port44444
log4j.appender.flumeAvro.UnsafeModetrue
log4j.appender.flumeAvro.layoutorg.apache.log4j.PatternLayout
log4j.appender.flumeAvro.layout.ConversionPattern%m
# set root logger
log4j.rootLoggerINFO, flumeAvro ISALog.java package cn.sinobest.asj.log;
import javax.jws.WebParam;
import javax.jws.WebService;
import cn.sinobest.asj.log.exception.InvalidFormatExceptioin;
import cn.sinobest.asj.log.exception.InvalidGradeException;
/*** SINOBEST ASJ Log - 为实现日志的统一采集和管理.* * author lijinlong* */
WebService
public interface ISALog {/*** 日志记录.* * param grade* 日志等级描述 - 忽略大小写.* param content* 日志内容 - 需要为JSON格式的字符串.*/public void log(WebParam(name grade) String grade,WebParam(name content) String content)throws InvalidGradeException, InvalidFormatExceptioin;
} SALogImpl.java 1 package cn.sinobest.asj.log;2 import javax.jws.WebService;3 import org.apache.commons.logging.Log;4 import org.apache.commons.logging.LogFactory;5 import cn.sinobest.asj.log.exception.InvalidFormatExceptioin;6 import cn.sinobest.asj.log.exception.InvalidGradeException;7 import cn.sinobest.asj.log.util.MessageTemplate;8 import cn.sinobest.asj.log.util.ValidGrade;9 import com.mongodb.util.JSON;10 WebService(endpointInterface cn.sinobest.asj.log.ISALog)11 public class SALogImpl implements ISALog {12 static final Log log LogFactory.getLog(SALogImpl.class);13 public void log(String grade, String content) throws InvalidGradeException,14 InvalidFormatExceptioin {15 checkGrade(grade);16 checkContent(content);17 ValidGrade vg ValidGrade.valueOf(grade.toUpperCase());18 log(vg, content);19 }20 /**21 * 根据日志等级调用{link log}的不同方法记录日志.22 * 23 * param vg24 * 日志等级25 * param content26 * 日志内容27 */28 private void log(ValidGrade vg, String content) {29 switch (vg) {30 case DEBUG:31 log.debug(content);32 break;33 case INFO:34 log.info(content);35 break;36 case WARN:37 log.warn(content);38 break;39 case ERROR:40 log.error(content);41 break;42 default:43 break;44 }45 }46 /**47 * 检查日志等级的有效性.48 * 49 * param grade50 * 日志等级描述.51 * throws InvalidGradeException52 * 当日志等级无效时抛出此异常.53 */54 private void checkGrade(String grade) throws InvalidGradeException {55 boolean valid ValidGrade.isValid(grade);56 if (!valid) {57 String message String.format(MessageTemplate.INVALID_GRADE,58 grade, ValidGrade.getEnumContent());59 throw new InvalidGradeException(message);60 }61 }62 /**63 * 检查日志内容格式的有效性.br64 * 要求为JSON格式的字符串.65 * 66 * param content67 * 日志内容.68 * throws InvalidFormatExceptioin69 * 当日志内容格式无效时抛出此异常.70 */71 private void checkContent(String content) throws InvalidFormatExceptioin {72 boolean valid true;73 if (content null || content.isEmpty()) {74 valid false;75 } else {76 try {77 JSON.parse(content);78 valid true;79 } catch (com.mongodb.util.JSONParseException e) {80 valid false;81 }82 }83 if (!valid) {84 String message String.format(MessageTemplate.INVALID_FORMAT,85 content);86 throw new InvalidFormatExceptioin(message);87 }88 }89 /**90 * just for test.91 * 92 * param args93 */94 public static void main(String[] args) {95 String[][] data { { info, {name:ljl,age:26} },96 { INFO, trouble is a friend. },97 { JOKE, {message:I am feeling down.} } };98 ISALog ilog new SALogImpl();99 for (String[] dat : data) {
100 String grade dat[0];
101 String content dat[1];
102 try {
103 ilog.log(grade, content);
104 } catch (Exception e) {
105 e.printStackTrace();
106 }
107 }
108 }
109 } SALogImpl.java InvalidGradeException.java package cn.sinobest.asj.log.exception;
/*** 表示无效的日志等级.* author lijinlong**/
public class InvalidGradeException extends Exception {private static final long serialVersionUID 1341726127995938030L;public InvalidGradeException(String message) {super(message);}
} InvalidFormatExceptioin.java package cn.sinobest.asj.log.exception;
/*** 表示无效的日志等级.* author lijinlong**/
public class InvalidGradeException extends Exception {private static final long serialVersionUID 1341726127995938030L;public InvalidGradeException(String message) {super(message);}
} ValidGrade.java 1 package cn.sinobest.asj.log.util;2 /**3 * 有效的日志等级.4 * 5 * author lijinlong6 * 7 */8 public enum ValidGrade {9 DEBUG, INFO, WARN, ERROR;
10 /** 有效日志等级的枚举内容 */
11 private static String enumContent;
12 /**
13 * 获取所有有效的日志等级.
14 *
15 * return
16 */
17 public static String getEnumContent() {
18 if (enumContent ! null !enumContent.isEmpty())
19 return enumContent;
20 ValidGrade[] vgs ValidGrade.values();
21 StringBuilder builder new StringBuilder(30);
22 for (ValidGrade vg : vgs) {
23 builder.append(vg).append(,);
24 }
25 builder.delete(builder.length() - 1, builder.length());
26 enumContent builder.toString();
27 return enumContent;
28 }
29
30 /**
31 * 判断日志等级是否有效.
32 * param grade 日志等级 - 忽略大小写.
33 * return
34 */
35 public static boolean isValid(String grade) {
36 if (grade null || grade.isEmpty())
37 return false;
38
39 boolean result false;
40
41 final String GRADE grade.toUpperCase();
42 ValidGrade[] vgs ValidGrade.values();
43 for (ValidGrade vg : vgs) {
44 if (vg.toString().equals(GRADE)) {
45 result true;
46 break;
47 }
48 }
49
50 return result;
51 }
52
53 /**
54 * just for test.
55 * param args
56 */
57 public static void main(String[] args) {
58 String content getEnumContent();
59 System.out.println(content);
60
61 String[] testGrade {DEBUG, INFO, WARN, ERROR, TEST};
62 for (String tg : testGrade) {
63 if (!ValidGrade.isValid(tg)) {
64 String message String.format(%s is invalid., tg);
65 System.out.println(message);
66 }
67 }
68 }
69 } ValidGrade.java MessageTemplate.java package cn.sinobest.asj.log.util;
/*** 消息模板.* author lijinlong**/
public class MessageTemplate {/** 无效的消息等级 */public static final String INVALID_GRADE 无效的日志等级[%s]。服务支持的日志等级有%s。;/** 无效的消息内容格式 */public static final String INVALID_FORMAT 无效的日志内容格式:\n%s\n请检查是否为JSON格式的字符串。;
} sun-jaxws.xml ?xml version1.0 encodingUTF-8?
endpoints xmlnshttp://java.sun.com/xml/ns/jax-ws/ri/runtimeversion2.0endpoint namedefaultLog implementationcn.sinobest.asj.log.SALogImplurl-pattern/log.action /
/endpoints 应用系统群 1.需要考虑的问题 1.1 拦截 使用Filter可以实现拦截。 1.2 组织日志内容 视需求而定当前仅对request中的部分信息进行了采集。 1.3 格式化 日志信息需要格式化为JSON字符串才能正确的写到MongoDB。 1.4 请求webservice服务 参考基于wsimport生成代码的客户端 - 外网参考基于wsimport生成代码的客户端 2. demo 2.1 文件结构图 src|---- cn.sinobest.asj.log|---- LogFilter.java|---- cn.sinobest.asj.log.wsimport # 存放wsimport生成的代码# 省略
basic|---- WEB-INF|---- web.xml 2.2 文件内容 LogFilter.java 1 package cn.sinobest.asj.log;2 import java.io.IOException;3 import java.net.MalformedURLException;4 import java.net.URL;5 import java.util.Date;6 import java.util.HashMap;7 import java.util.Map;8 import javax.servlet.Filter;9 import javax.servlet.FilterChain;
10 import javax.servlet.FilterConfig;
11 import javax.servlet.ServletException;
12 import javax.servlet.ServletRequest;
13 import javax.servlet.ServletResponse;
14 import javax.servlet.http.HttpServletRequest;
15 import org.apache.commons.logging.Log;
16 import org.apache.commons.logging.LogFactory;
17 import org.json.JSONObject;
18 import cn.sinobest.asj.log.wsimport.ISALog;
19 import cn.sinobest.asj.log.wsimport.InvalidFormatExceptioin_Exception;
20 import cn.sinobest.asj.log.wsimport.InvalidGradeException_Exception;
21 import cn.sinobest.asj.log.wsimport.SALogImplService;
22 public class LogFilter implements Filter {
23 static final Log log LogFactory.getLog(LogFilter.class);
24 static final String WSDL_LOCATION http://localhost:8080/logserv/log.action?wsdl;
25 Override
26 public void destroy() {
27 }
28 Override
29 public void doFilter(ServletRequest request, ServletResponse response,
30 FilterChain chain) throws IOException, ServletException {
31 try {
32 log(request);
33 } catch (InvalidFormatExceptioin_Exception e) {
34 e.printStackTrace();
35 } catch (InvalidGradeException_Exception e) {
36 e.printStackTrace();
37 } finally {
38 chain.doFilter(request, response);
39 }
40 }
41 private void log(ServletRequest request) throws MalformedURLException,
42 InvalidFormatExceptioin_Exception, InvalidGradeException_Exception {
43 MapString, Object data new HashMapString, Object();
44 data.put(appid, zfba);
45 data.put(time, new Date());
46 data.put(localAddr, request.getLocalAddr());
47 data.put(localName, request.getLocalName());
48 data.put(localPort, request.getLocalPort());
49 data.put(remoteAddr, request.getRemoteAddr());
50 data.put(remoteHost, request.getRemoteHost());
51 data.put(remotePort, request.getRemotePort());
52 // data.put(serverName, request.getServerName());
53 // data.put(serverPort, request.getServerPort());
54 HttpServletRequest hrequest (HttpServletRequest) request;
55 data.put(pathInfo, hrequest.getPathInfo());
56 data.put(pathTranslated, hrequest.getPathTranslated());
57 data.put(remoteUser, hrequest.getRemoteUser());
58 data.put(requestURI, hrequest.getRequestURI());
59 data.put(requestURL, hrequest.getRequestURL());
60 data.put(servletPath, hrequest.getServletPath());
61 JSONObject cont new JSONObject(data);
62 URL url new URL(WSDL_LOCATION);
63 SALogImplService ss new SALogImplService(url);
64 ISALog service ss.getSALogImplPort();
65 service.log(info, cont.toString());
66 }
67 Override
68 public void init(FilterConfig arg0) throws ServletException {
69 }
70 } LogFilter.java web.xml这里仅贴出新增的内容 !-- 测试日志 --filterfilter-namelog-filter/filter-namefilter-classcn.sinobest.asj.log.LogFilter/filter-class/filter!-- 测试日志 --filter-mappingfilter-namelog-filter/filter-nameurl-pattern/*/url-pattern/filter-mapping 测试 启动MongoDB参考《Flume学习应用Java写日志数据到MongoDB》配置并启动Flume参考《Flume学习应用Java写日志数据到MongoDB》启动日志服务参考《在web项目中发布jaxws》启动应用系统并进行访问查看MongoDB数据库参考《Flume学习应用Java写日志数据到MongoDB》附录 相关文章 Flume学习应用Java写日志数据到MongoDB博客园Flume学习应用Java写日志数据到MongoDB在web项目中发布jaxws博客园在web项目中发布jaxws基于wsimport生成代码的客户端博客园基于wsimport生成代码的客户端 转载于:https://www.cnblogs.com/ywjy/p/5259291.html
