Dolphinscheduler2.0.0可视化⼯作流任务调度平台部署Dolphinscheduler 下载：
wget --no-check-certificate /dolphinscheduler/2.0.0-alpha/apache-dolphinscheduler-2.0.
Dolphinscheduler 集部署
前置准备⼯作 && 准备 DolphinScheduler 启动环境
1. JDK（1.8+）
2. mysql5.7
3. zk
4. yum install psmisc #进程树分析
配置⽤户和权限
# 创建⽤户需使⽤ root 登录
useradd dolphinscheduler
# 添加密码
echo "dolphinscheduler" | passwd --stdin dolphinscheduler
# 配置 sudo 免密
sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
sed -i 's/Defaults requirett/#Defaults requirett/g' /etc/sudoers
# 修改⽬录权限，使得部署⽤户对 dolphinscheduler-bin ⽬录有操作权限
chown -R dolphinscheduler:dolphinscheduler dolphinscheduler-bin
密钥
su dolphinscheduler
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 600 ~/.ssh/authorized_keys
初始化数据库
mysql -uroot -p
mysql> CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
mysql> CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler';
mysql> GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%' ;
mysql> source /usr/local/src/apache-dolphinscheduler/sql/dolphinscheduler_mysql.sql;
mysql> flush privileges;
拷贝 mysql 驱动程序
mysql-connector-java 到 DolphinScheduler lib⽬录下
[dolphinscheduler@hdp0 ~]$ scp mysql-connector-java.jar dolphinscheduler@hdp1:/data/apache-dolphinscheduler/lib/
修改配置
进⼊到 /data/tools/apache-dolphinscheduler/conf/config
下载apache修改f
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# /licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IP: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
ips="hdp0,hdp1,hdp2,hdp3"
# Port of SSH protocol, default value is22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
sshPort="22"
# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds1,ds2", Example for IP: ips="192.168.8.1,192.168.8.2"
masters="hdp0,hdp1"
# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: ips="ds1:default,ds2:default,ds3:default", Example for IP: ips="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
workers="hdp2:default,hdp3:default"
# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds3", Example for IP: ips="192.168.8.3"
alertServer="hdp0"
# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostnames: ips="ds1", Example for IP: ips="192.168.8.1"
apiServers="hdp1,hdp0"
# The directory to install DolphinScheduler for all machine we config above. It will automatically created by `install.sh` script if not exists.
# **DO NOT** set this configuration same as the current path (pwd)
#installPath="/data1_1T/dolphinscheduler"
installPath="/data/dolphinscheduler"
# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before run `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser="dolphinscheduler"
# The directory to store local data for all machine we config above. Make sure user `deployUser` have permissions to read and write this directory.
#dataBasedirPath="/tmp/dolphinscheduler"
dataBasedirPath="/data/dolphinscheduler/dolphinschedulerdata"
# ---------------------------------------------------------
# DolphinScheduler ENV
# ---------------------------------------------------------
# JAVA_HOME, we recommend use same JAVA_HOME in all machine you going to install DolphinS
cheduler
# and this configuration only support one parameter so far.
javaHome="/data/jdk"
# DolphinScheduler API service port, also this your DolphinScheduler UI component's URL port, default values is 12345
apiServerPort="12345"
# ---------------------------------------------------------
# Database
# NOTICE: If database value has special characters, such as `.*[]^${}\+?|()@#&`, Please add prefix `\` for escaping.
# ---------------------------------------------------------
# The type for the metadata database
# Supported values: ``postgresql``, ``mysql``.
dbtype="mysql"
# The <HOST>:<PORT> connection pair DolphinScheduler connect to the metadata database
dbhost="172.31.115.17:3306"
# The username DolphinScheduler connect to the metadata database
username="dolphinscheduler"
# The password DolphinScheduler connect to the metadata database
password="dolphinscheduler"
# The database DolphinScheduler connect to the metadata database
dbname="dolphinscheduler"
# ---------------------------------------------------------
# Registry Server
# ---------------------------------------------------------
# Registry Server plugin dir. DolphinScheduler will find and load the registry plugin jar package from this dir.
# For now default registry server is zookeeper, so the default value is `lib/plugin/registry/zookeeper`.
# If you want to implement your own registry server, please see /en-us/docs/dev/user_doc/registry_spi.html
registryPluginDir="lib/plugin/registry/zookeeper"
# Registry Server plugin name, should be a substring of `registryPluginDir`, DolphinScheduler use this for verifying configuration consistency
registryPluginName="zookeeper"
# Registry Server address.
registryServers="172.31.115.17:2181,172.31.115.18:2181,172.31.115.19:2181"
# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot="/dolphinscheduler"
# ---------------------------------------------------------
# Alert Server
# ---------------------------------------------------------
# Alert Server plugin dir. DolphinScheduler will find and load the alert plugin jar package from this dir.
alertPluginDir="lib/plugin/alert"
# ---------------------------------------------------------
# Worker Task Server
# ---------------------------------------------------------
# Worker Task Server plugin dir. DolphinScheduler will find and load the worker task plugin jar package from this dir.
taskPluginDir="lib/plugin/task"
# resource storage type: HDFS, S3, NONE
resourceStorageType="HDFS"
# resource store on HDFS/S3 path, resource file will store to this hadoop hdfs path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler"is recommended resourceUploadPath="/dolphinscheduler"
# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to l l in the conf directory.
# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
# Note，s3 be sure to create the root directory /dolphinscheduler
defaultFS="hdfs://hdp0.fengjian:8020"
# if resourceStorageType is S3, the following three configuration is required, otherwise please ignore
s3Endpoint=":9010"
s3AccessKey="xxxxxxxxxx"
s3SecretKey="xxxxxxxxxx"
# resourcemanager port, the default value is8088if not specified
resourceManagerHttpAddressPort="8088"
# if resourcemanager HA is enabled, please set the HA IPs; if resourcemanager is single, keep this value empty
yarnHaIps="hdp0.fengjian,hdp1.fengjian"
# if resourcemanager HA is enabled or not use resourcemanager, please keep the default value; If resourcemanager is single, you only need to replace ds1 to actual resourcemanager hostname singleYarnIp=""
# who have permissions to create directory under HDFS/S3 root path
# Note: if kerberos is enabled, please config hdfsRootUser=
hdfsRootUser="hdfs"
# kerberos config
# whether kerberos starts, if kerberos starts, following four items need to config, otherwise please ignore
kerberosStartUp="false"
# kdc krb5 config file path
krb5ConfPath="$installPath/f"
# keytab username,watch out the @ sign should followd by \\
keytabUserName="hdfs-mycluster\\@ESZ.COM"
# username keytab path
keytabPath="$installPath/conf/hdfs.headless.keytab"
# kerberos expire time, the unit is hour
kerberosExpireTime="2"
# use sudo or not
sudoEnable="true"
# worker tenant auto create
workerTenantAutoCreate="false"
如果是hdfs集
# if resourceStorageType is HDFS，defaultFS write namenode address，HA you need to l l in the conf directory.
# if S3，write S3 address，HA，for example ：s3a://dolphinscheduler，
# Note，s3 be sure to create the root directory /dolphinscheduler
defaultFS="hdfs://Abcd:8020"
并且拷贝
cp /data/hadoop/l /usr/local/src/dolphscheduler/config/
cp /data/hadoop/etc/hdfs
-l /usr/local/src/dolphscheduler/config/
修改环境变量
进⼊到 /data/tools/apache-dolphinscheduler/conf/env
修改dolphinscheduler_env.sh
export HADOOP_HOME=/usr/hdp/3.0.0.0-1634/hadoop
export HADOOP_CONF_DIR=/usr/hdp/3.0.0.0-1634/hadoop/etc/hadoop
export SPARK_HOME1=/usr/hdp/3.0.0.0-1634/spark2
export SPARK_HOME2=/usr/hdp/3.0.0.0-1634/spark2
#export PYTHON_HOME=/opt/soft/python
export JAVA_HOME=/data/jdk
export HIVE_HOME=/usr/hdp/3.0.0.0-1634/hive
#export FLINK_HOME=/opt/soft/flink
#export DATAX_HOME=/opt/soft/datax
export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$PATH
#export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$PATH
修改jvm 堆栈参数
/data/tools/apache-dolphinscheduler/bin/dolphinscheduler-daemon.sh
根据内存修改
启动 Dolphinscheduler
#会复制到其他服务器，并且启动
sh install.sh
登陆Dolphinscheduler
浏览器访问地址 hdp0.fengjjian:12345/dolphinscheduler 即可登录系统UI。默认的⽤户名和密码是 admin/dolphinscheduler123
⼀些其他的启停服务命令
# ⼀键停⽌集所有服务
sh ./bin/stop-all.sh
# ⼀键开启集所有服务
sh ./bin/start-all.sh
# 启停 Master
sh ./bin/dolphinscheduler-daemon.sh stop master-server sh ./bin/dolphinscheduler-daemon.sh start master-server # 启停 Worker
sh ./bin/dolphinscheduler-daemon.sh start worker-server sh ./bin/dolphinscheduler-daemon.sh stop worker-server # 启停 Api
sh ./bin/dolphinscheduler-daemon.sh start api-server
sh ./bin/dolphinscheduler-daemon.sh stop api-server
# 启停 Logger
sh ./bin/dolphinscheduler-daemon.sh start logger-server sh ./bin/dolphinscheduler-daemon.sh stop logger-server # 启停 Alert
sh ./bin/dolphinscheduler-daemon.sh start alert-server sh ./bin/dolphinscheduler-daemon.sh stop alert-server
版权声明：本站内容均来自互联网，仅供演示用，请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系QQ:729038198，我们将在24小时内删除。
发表评论