2016-10-20-flume-1.7

download

1	wget https://apache.fayea.com/flume/1.7.0/apache-flume-1.7.0-bin.tar.gz

document

https://flume.apache.org/FlumeUserGuide.html#flume-1-7-0-user-guide

config

hdfs flume-conf.properties

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#  https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.


# The configuration file needs to define the sources,
# the channels and the sinks.
# Sources, channels and sinks are defined per agent,
# in this case called 'agent'

agent.sources = execSource
agent.channels = memoryChannel
agent.sinks = hdfsSink

# For each one of the sources, the type is defined
# agent.sources.seqGenSrc.type = seq
agent.sources.execSource.type = exec
agent.sources.execSource.command = tail -F /usr/local/tomcat/logs/catalina.out
agent.sources.execSource.channels = memoryChannel
# The channel can be defined as follows.
# agent.sources.seqGenSrc.channels = memoryChannel

# Each sink's type must be defined
# agent.sinks.loggerSink.type = logger
# agent.sinks.fileRollSink.type = file_roll
# agent.sinks.fileRollSink.channel = memoryChannel
# agent.sinks.fileRollSink.sink.directory = /var/log/flume
agent.sinks.hdfsSink.type = hdfs
agent.sinks.hdfsSink.channel = memoryChannel
#agent.sinks.hdfsSink.hdfs.path = hdfs://192.168.88.21:8020/flume
agent.sinks.hdfsSink.hdfs.path = hdfs://192.168.88.180:9000/flume
agent.sinks.hdfsSink.hdfs.fileType = DataStream

kafka.properties

a1.sources = source1
a1.channels = channel1
a1.sinks = sink1

a1.sources.source1.type = exec
a1.sources.source1.command = tail -F /opt/log/catalina.out
# a1.sources.source1.command = tail -F /usr/local/nginx/logs/access.log
a1.sources.source1.channels = channel1

a1.channels.channel1.type = SPILLABLEMEMORY
a1.channels.channel1.memoryCapacity = 10000
a1.channels.channel1.overflowCapacity = 1000000
a1.channels.channel1.byteCapacity = 800000
a1.channels.channel1.checkpointDir = /mnt/flume/checkpoint
a1.channels.channel1.dataDirs = /mnt/flume/data

a1.sinks.sink1.type = org.apache.flume.sink.kafka.KafkaSink
a1.sinks.sink1.kafka.topic = log
a1.sinks.sink1.kafka.bootstrap.servers = 192.168.100.34:9092,192.168.100.35:9092
a1.sinks.sink1.kafka.producer.acks = 1
a1.sinks.sink1.flumeBatchSize = 20
a1.sinks.sink1.channel = channel1

flume-env.sh

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced
# during Flume startup.


# Enviroment variables can be set here.

# export JAVA_HOME=/usr/lib/jvm/java-6-sun
export JAVA_HOME=/usr/java

# Give Flume more memory and pre-allocate, enable remote monitoring via JMX
# export JAVA_OPTS="-Xms100m -Xmx2000m -Dcom.sun.management.jmxremote"
export HADOOP_HOME=/usr/local/hadoop-2.3.0
# Note that the Flume conf directory is always included in the classpath.
FLUME_CLASSPATH="$HADOOP_HOME/share/hadoop/hdfs/hadoop-hdfs-2.3.0.jar"

启动

1	nohup bin/flume-ng agent --conf conf --conf-file conf/kafka.cfg --name a1 -Dflume.root.logger=INFO,console &