openstack/openstack-helm
Code Issues Proposed changes

[neutron] Unhardcode probes timings

Browse Source 
This patch adds ability to unhardcode readiness/
liveness probes timings for those probes which still
were hardcoded. Moreover it introduces
RPC_PROBE_TIMEOUT and RPC_PROBE_RETRIES variables
which are passed to health probe script and
allow to unhardcode RPCtest  timeout and number of
retries

Change-Id: I2e48eed26abb82208a4ac4ae596d27ca8db99c90
This commit is contained in:
Oleksii Grudev 2020-02-10 17:41:40 +02:00
parent 4a5374aebd
commit 6969a5d596
9 changed files with 106 additions and 44 deletions

6
neutron/templates/bin/_health-probe.py.tpl

@ -51,6 +51,8 @@ from oslo_context import context
from oslo_log import log
import oslo_messaging
rpc_timeout = int(os.getenv('RPC_PROBE_TIMEOUT', '60'))
rpc_retries = int(os.getenv('RPC_PROBE_RETRIES', '2'))
rabbit_port = 5672
tcp_established = "ESTABLISHED"
log.logging.basicConfig(level=log.ERROR)
@ -69,8 +71,8 @@ def check_agent_status(transport):
topic=cfg.CONF.agent_queue_name,
server=_get_hostname(use_fqdn))
client = oslo_messaging.RPCClient(transport, target,
timeout=60,
retry=2)
timeout=rpc_timeout,
retry=rpc_retries)
client.call(context.RequestContext(),
'pod_health_probe_method_ignore_errors')
except oslo_messaging.exceptions.MessageDeliveryFailure:

19
neutron/templates/daemonset-bagpipe-bgp.yaml

@ -14,6 +14,16 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "bagpipeBgpLivenessProbeTemplate" }}
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
{{- define "bagpipeBgpReadinessProbeTemplate" }}
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
{{- end }}
{{- define "neutron.bagpipe_bgp.daemonset" }}
{{- $daemonset := index . 0 }}
{{- $configMapName := index . 1 }}
@ -81,13 +91,8 @@ spec:
{{ tuple $envAll "neutron_bagpipe_bgp" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.bagpipe_bgp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_bagpipe_bgp" "container" "neutron_bagpipe_bgp" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
readinessProbe:
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
livenessProbe:
tcpSocket:
port: {{ tuple "network" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}
initialDelaySeconds: 60
{{ dict "envAll" $envAll "component" "bagpipe_bgp" "container" "bagpipe_bgp" "type" "liveness" "probeTemplate" (include "bagpipeBgpLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "bagpipe_bgp" "container" "bagpipe_bgp" "type" "readiness" "probeTemplate" (include "bagpipeBgpReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:
- /tmp/neutron-bagpipe-bgp.sh
volumeMounts:

5
neutron/templates/daemonset-dhcp-agent.yaml

@ -157,6 +157,11 @@ spec:
{{ tuple $envAll "neutron_dhcp" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.dhcp | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_dhcp_agent" "container" "neutron_dhcp_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "readiness" "probeTemplate" (include "dhcpAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "dhcp_agent" "container" "dhcp_agent" "type" "liveness" "probeTemplate" (include "dhcpAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:

75
neutron/templates/daemonset-l2gw-agent.yaml

@ -14,6 +14,39 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "l2gwAgentLivenessProbeTemplate" }}
exec:
command:
- python
- /tmp/health-probe.py
- --config-file
- /etc/neutron/neutron.conf
- --config-file
- /etc/neutron/l2gw_agent.ini
- --agent-queue-name
- l2gateway_agent
- --liveness-probe
{{- if .Values.pod.use_fqdn.neutron_agent }}
- --use-fqdn
{{- end }}
{{- end }}
{{- define "l2gwAgentReadinessProbeTemplate" }}
exec:
command:
- python
- /tmp/health-probe.py
- --config-file
- /etc/neutron/neutron.conf
- --config-file
- /etc/neutron/l2gw_agent.ini
- --agent-queue-name
- l2gateway_agent
{{- if .Values.pod.use_fqdn.neutron_agent }}
- --use-fqdn
{{- end }}
{{- end }}
{{- define "neutron.l2gw_agent.daemonset" }}
{{- $daemonset := index . 0 }}
{{- $configMapName := index . 1 }}
@ -66,41 +99,13 @@ spec:
{{ tuple $envAll $envAll.Values.pod.resources.agent.l2gw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
securityContext:
privileged: true
readinessProbe:
exec:
command:
- python
- /tmp/health-probe.py
- --config-file
- /etc/neutron/neutron.conf
- --config-file
- /etc/neutron/l2gw_agent.ini
- --agent-queue-name
- l2gateway_agent
{{- if .Values.pod.use_fqdn.neutron_agent }}
- --use-fqdn
{{- end }}
initialDelaySeconds: 30
periodSeconds: 190
timeoutSeconds: 185
livenessProbe:
exec:
command:
- python
- /tmp/health-probe.py
- --config-file
- /etc/neutron/neutron.conf
- --config-file
- /etc/neutron/l2gw_agent.ini
- --agent-queue-name
- l2gateway_agent
- --liveness-probe
{{- if .Values.pod.use_fqdn.neutron_agent }}
- --use-fqdn
{{- end }}
initialDelaySeconds: 120
periodSeconds: 600
timeoutSeconds: 580
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "l2gw_agent" "container" "l2gw_agent" "type" "liveness" "probeTemplate" (include "l2gwAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "l2gw_agent" "container" "l2gw_agent" "type" "readiness" "probeTemplate" (include "l2gwAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:
- /tmp/neutron-l2gw-agent.sh
volumeMounts:

5
neutron/templates/daemonset-l3-agent.yaml

@ -158,6 +158,11 @@ spec:
{{ tuple $envAll "neutron_l3" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.l3 | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_l3_agent" "container" "neutron_l3_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "readiness" "probeTemplate" (include "l3AgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "l3_agent" "container" "l3_agent" "type" "liveness" "probeTemplate" (include "l3AgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:

5
neutron/templates/daemonset-metadata-agent.yaml

@ -115,6 +115,11 @@ spec:
- name: neutron-metadata-agent
{{ tuple $envAll "neutron_metadata" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "readiness" "probeTemplate" (include "metadataAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "metadata_agent" "container" "metadata_agent" "type" "liveness" "probeTemplate" (include "metadataAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
securityContext:

5
neutron/templates/daemonset-ovs-agent.yaml

@ -179,6 +179,11 @@ spec:
- name: neutron-ovs-agent
{{ tuple $envAll "neutron_openvswitch_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.ovs | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "readiness" "probeTemplate" (include "ovsAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "component" "ovs_agent" "container" "ovs_agent" "type" "liveness" "probeTemplate" (include "ovsAgentLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_ovs_agent" "container" "neutron_ovs_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}

5
neutron/templates/daemonset-sriov-agent.yaml

@ -137,6 +137,11 @@ spec:
{{ tuple $envAll "neutron_sriov_agent" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.agent.sriov | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
{{ dict "envAll" $envAll "application" "neutron_sriov_agent" "container" "neutron_sriov_agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: RPC_PROBE_TIMEOUT
value: "{{ .Values.pod.probes.rpc_timeout }}"
- name: RPC_PROBE_RETRIES
value: "{{ .Values.pod.probes.rpc_retries }}"
{{ dict "envAll" $envAll "component" "sriov_agent" "container" "sriov_agent" "type" "readiness" "probeTemplate" (include "sriovAgentReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }}
command:
- /tmp/neutron-sriov-agent.sh

25
neutron/values.yaml

@ -339,6 +339,8 @@ pod:
use_fqdn:
neutron_agent: true
probes:
rpc_timeout: 60
rpc_retries: 2
dhcp_agent:
dhcp_agent:
readiness:
@ -404,6 +406,29 @@ pod:
initialDelaySeconds: 30
periodSeconds: 190
timeoutSeconds: 185
bagpipe_bgp:
bagpipe_bgp:
readiness:
enabled: true
params:
liveness:
enabled: true
params:
initialDelaySeconds: 60
l2gw_agent:
l2gw_agent:
readiness:
enabled: true
params:
initialDelaySeconds: 30
periodSeconds: 15
timeoutSeconds: 65
liveness:
enabled: true
params:
initialDelaySeconds: 120
periodSeconds: 90
timeoutSeconds: 70
server:
server:
readiness: