add network filter support in native plugin (#627)

Signed-off-by: spacewander <[email protected]>

controller/internal/istio/envoyfilter.go

@@ -206,89 +206,67 @@ func GenerateRouteFilter(host *model.VirtualHost, route string, config map[strin
 	}
 }
 
-func GenerateLDSFilterViaECDS(key string, ldsName string, config map[string]interface{}) *istiov1a3.EnvoyFilter {
+func GenerateLDSFilterViaECDS(key string, ldsName string, hasHCM bool, config map[string]interface{}) *istiov1a3.EnvoyFilter {
 	ef := &istiov1a3.EnvoyFilter{
 		Spec: istioapi.EnvoyFilter{},
 	}
 
-	// Always create a Go filter for ECDS so we don't trigger LDS drain when adding/removing Go Plugins.
-	// We can't do this for native filters now.
-	// For TCP proxy use case, the extra golang-filter is harmless, because it only attaches to HCM.
-	cfg := config[model.ECDSGolangFilter]
-	if cfg == nil {
-		cfg = map[string]interface{}{}
-	}
-	ecdsName := key + "-" + model.GolangPluginsFilter
-	ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
-		&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
-			ApplyTo: istioapi.EnvoyFilter_HTTP_FILTER,
-			Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
-				ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
-					Listener: &istioapi.EnvoyFilter_ListenerMatch{
-						Name: ldsName,
-						FilterChain: &istioapi.EnvoyFilter_ListenerMatch_FilterChainMatch{
-							Filter: &istioapi.EnvoyFilter_ListenerMatch_FilterMatch{
-								Name: "envoy.filters.network.http_connection_manager",
-								SubFilter: &istioapi.EnvoyFilter_ListenerMatch_SubFilterMatch{
-									Name: "htnn.filters.http.golang",
-								},
+	if config[model.ECDSListenerFilter] != nil {
+		cfg, _ := config[model.ECDSListenerFilter].([]*fmModel.FilterConfig)
+		for i := len(cfg) - 1; i >= 0; i-- {
+			filter := cfg[i]
+			ecdsName := key + "-" + filter.Name
+			c, _ := filter.Config.(map[string]interface{})
+			typeURL, _ := c["@type"].(string)
+			ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
+				&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+					ApplyTo: istioapi.EnvoyFilter_LISTENER_FILTER,
+					Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
+						ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
+							Listener: &istioapi.EnvoyFilter_ListenerMatch{
+								Name: ldsName,
 							},
 						},
 					},
-				},
-			},
-			Patch: &istioapi.EnvoyFilter_Patch{
-				Operation: istioapi.EnvoyFilter_Patch_INSERT_BEFORE,
-				Value: MustNewStruct(map[string]interface{}{
-					"name": ecdsName,
-					"config_discovery": map[string]interface{}{
-						"apply_default_config_without_warming": true,
-						"default_config": map[string]interface{}{
-							"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
-							"library_id":   "fm",
-							"library_path": ctrlcfg.GoSoPath(),
-							"plugin_name":  "fm",
-						},
-						"config_source": map[string]interface{}{
-							"ads": map[string]interface{}{},
-						},
-						"type_urls": []interface{}{
-							"type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
-						},
+					Patch: &istioapi.EnvoyFilter_Patch{
+						Operation: istioapi.EnvoyFilter_Patch_INSERT_FIRST,
+						Value: MustNewStruct(map[string]interface{}{
+							"name": ecdsName,
+							"config_discovery": map[string]interface{}{
+								"config_source": map[string]interface{}{
+									"ads": map[string]interface{}{},
+								},
+								"type_urls": []interface{}{
+									typeURL,
+								},
+							},
+						}),
 					},
-				}),
-			},
-		},
-		&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
-			ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
-			Patch: &istioapi.EnvoyFilter_Patch{
-				Operation: istioapi.EnvoyFilter_Patch_ADD,
-				Value: MustNewStruct(map[string]interface{}{
-					"name": ecdsName,
-					"typed_config": map[string]interface{}{
-						"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
-						"library_id":   "fm",
-						"library_path": ctrlcfg.GoSoPath(),
-						"plugin_name":  "fm",
-						"plugin_config": map[string]interface{}{
-							"@type": "type.googleapis.com/xds.type.v3.TypedStruct",
-							"value": cfg,
-						},
+				},
+				&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+					ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
+					Patch: &istioapi.EnvoyFilter_Patch{
+						Operation: istioapi.EnvoyFilter_Patch_ADD,
+						Value: MustNewStruct(map[string]interface{}{
+							"name":         ecdsName,
+							"typed_config": filter.Config,
+						}),
 					},
-				}),
-			},
-		},
-	)
+				},
+			)
+		}
+	}
 
-	if config[model.ECDSListenerFilter] != nil {
-		cfg, _ := config[model.ECDSListenerFilter].([]*fmModel.FilterConfig)
-		for _, filter := range cfg {
+	if config[model.ECDSNetworkFilter] != nil {
+		cfg, _ := config[model.ECDSNetworkFilter].([]*fmModel.FilterConfig)
+		for i := len(cfg) - 1; i >= 0; i-- {
+			filter := cfg[i]
 			ecdsName := key + "-" + filter.Name
 			c, _ := filter.Config.(map[string]interface{})
 			typeURL, _ := c["@type"].(string)
 			ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
 				&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
-					ApplyTo: istioapi.EnvoyFilter_LISTENER_FILTER,
+					ApplyTo: istioapi.EnvoyFilter_NETWORK_FILTER,
 					Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
 						ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
 							Listener: &istioapi.EnvoyFilter_ListenerMatch{
@@ -325,6 +303,74 @@ func GenerateLDSFilterViaECDS(key string, ldsName string, config map[string]inte
 		}
 	}
 
+	if hasHCM {
+		cfg := config[model.ECDSGolangFilter]
+		if cfg == nil {
+			cfg = map[string]interface{}{}
+		}
+		ecdsName := key + "-" + model.GolangPluginsFilter
+		ef.Spec.ConfigPatches = append(ef.Spec.ConfigPatches,
+			&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+				ApplyTo: istioapi.EnvoyFilter_HTTP_FILTER,
+				Match: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch{
+					ObjectTypes: &istioapi.EnvoyFilter_EnvoyConfigObjectMatch_Listener{
+						Listener: &istioapi.EnvoyFilter_ListenerMatch{
+							Name: ldsName,
+							FilterChain: &istioapi.EnvoyFilter_ListenerMatch_FilterChainMatch{
+								Filter: &istioapi.EnvoyFilter_ListenerMatch_FilterMatch{
+									Name: "envoy.filters.network.http_connection_manager",
+									SubFilter: &istioapi.EnvoyFilter_ListenerMatch_SubFilterMatch{
+										Name: "htnn.filters.http.golang",
+									},
+								},
+							},
+						},
+					},
+				},
+				Patch: &istioapi.EnvoyFilter_Patch{
+					Operation: istioapi.EnvoyFilter_Patch_INSERT_BEFORE,
+					Value: MustNewStruct(map[string]interface{}{
+						"name": ecdsName,
+						"config_discovery": map[string]interface{}{
+							"apply_default_config_without_warming": true,
+							"default_config": map[string]interface{}{
+								"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+								"library_id":   "fm",
+								"library_path": ctrlcfg.GoSoPath(),
+								"plugin_name":  "fm",
+							},
+							"config_source": map[string]interface{}{
+								"ads": map[string]interface{}{},
+							},
+							"type_urls": []interface{}{
+								"type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+							},
+						},
+					}),
+				},
+			},
+			&istioapi.EnvoyFilter_EnvoyConfigObjectPatch{
+				ApplyTo: istioapi.EnvoyFilter_EXTENSION_CONFIG,
+				Patch: &istioapi.EnvoyFilter_Patch{
+					Operation: istioapi.EnvoyFilter_Patch_ADD,
+					Value: MustNewStruct(map[string]interface{}{
+						"name": ecdsName,
+						"typed_config": map[string]interface{}{
+							"@type":        "type.googleapis.com/envoy.extensions.filters.http.golang.v3alpha.Config",
+							"library_id":   "fm",
+							"library_path": ctrlcfg.GoSoPath(),
+							"plugin_name":  "fm",
+							"plugin_config": map[string]interface{}{
+								"@type": "type.googleapis.com/xds.type.v3.TypedStruct",
+								"value": cfg,
+							},
+						},
+					}),
+				},
+			},
+		)
+	}
+
 	return ef
 }
 

controller/internal/model/model.go

@@ -30,16 +30,26 @@ func (g GatewaySection) String() string {
 	return fmt.Sprintf("%s/%s", g.NsName.String(), g.SectionName)
 }
 
+type Gateway struct {
+	GatewaySection *GatewaySection
+	// HasHCM shows if the HCM HTTP filter is present in the gateway
+	HasHCM bool
+}
+
 type VirtualHost struct {
-	GatewaySection   *GatewaySection
-	NsName           *types.NamespacedName
-	Name             string
+	GatewaySection *GatewaySection
+	// NsName is the namespace and name of the k8s resource which generates VirtualHost
+	NsName *types.NamespacedName
+	// Name is the name of VirtualHost in the RDS
+	Name string
+	// ECDSResourceName is the name of ECDS which is used to configure the Gateway attached by this VirtualHost
 	ECDSResourceName string
 }
 
 const (
 	ECDSGolangFilter   = "golang"
 	ECDSListenerFilter = "listener"
+	ECDSNetworkFilter  = "network"
 
 	GolangPluginsFilter = "golang-filter"
 )

controller/internal/translation/data_plane_state.go

@@ -47,7 +47,7 @@ type routePolicy struct {
 }
 
 type gatewayPolicy struct {
-	NsName   *types.NamespacedName
+	Gateway  *model.Gateway
 	Policies []*FilterPolicyWrapper
 }
 
@@ -247,7 +247,13 @@ func addServerPortToProxy(gs *model.GatewaySection, serverPort ServerPort, proxi
 	}
 
 	gwPolicy := &gatewayPolicy{
-		NsName: &gs.NsName,
+		Gateway: &model.Gateway{
+			GatewaySection: gs,
+		},
+	}
+	switch serverPort.Protocol {
+	case "HTTP", "HTTPS", "GRPC", "GRPC-WEB", "HTTP2":
+		gwPolicy.Gateway.HasHCM = true
 	}
 	if len(policies) > 0 {
 		gwPolicy.Policies = policies

controller/internal/translation/final_state.go

@@ -75,6 +75,12 @@ func envoyFilterNameFromVirtualHost(vhost *model.VirtualHost) string {
 	return fmt.Sprintf("%s-%s.%s", prefix, vhost.NsName.Namespace, vhost.NsName.Name)
 }
 
+func envoyFilterNameFromLds(ldsName string) string {
+	ldsName = strings.ReplaceAll(ldsName, "_", "-")
+	ldsName = strings.ReplaceAll(ldsName, ":", "-")
+	return fmt.Sprintf("htnn-lds-%s", ldsName)
+}
+
 // finalState is the end of the translation. We convert the state to EnvoyFilter and write it to k8s.
 type FinalState struct {
 	EnvoyFilters map[component.EnvoyFilterKey]*istiov1a3.EnvoyFilter
@@ -125,10 +131,12 @@ func toFinalState(_ *Ctx, state *mergedState) (*FinalState, error) {
 				info = gateway.Policy.Info
 			}
 
-			ef := istio.GenerateLDSFilterViaECDS(key, name, config)
+			ef := istio.GenerateLDSFilterViaECDS(key, name, gateway.Gateay.HasHCM, config)
 			ef.SetNamespace(ns)
-			// Put all LDS level golang filters of the same namespace into the same EnvoyFilter.
-			ef.SetName(fmt.Sprintf("htnn-h-%s", ns))
+			// Put all LDS level filters of the same LDS into the same EnvoyFilter.
+			efName := envoyFilterNameFromLds(name)
+			// Each LDS has it own EnvoyFilter, so it's easy to figure out how many filters are inserted into one LDS and their order.
+			ef.SetName(efName)
 
 			efList = append(efList, &envoyFilterWrapper{
 				EnvoyFilter: ef,
@@ -138,7 +146,7 @@ func toFinalState(_ *Ctx, state *mergedState) (*FinalState, error) {
 	}
 
 	// Merge EnvoyFilters with same name. The number of EnvoyFilters is equal to the number of
-	// configured domains.
+	// configured domains and lds.
 	efws := map[component.EnvoyFilterKey]*envoyFilterWrapper{}
 	for _, ef := range efList {
 		key := component.EnvoyFilterKey{
@@ -170,29 +178,21 @@ func toFinalState(_ *Ctx, state *mergedState) (*FinalState, error) {
 		}
 		ef.Labels[constant.LabelCreatedBy] = "FilterPolicy"
 
-		// Sort here to avoid EnvoyFilter change caused by the order of ConfigPatch.
-		sort.Slice(ef.Spec.ConfigPatches, func(i, j int) bool {
-			a := ef.Spec.ConfigPatches[i]
-			b := ef.Spec.ConfigPatches[j]
-			aName := a.Patch.Value.AsMap()["name"]
-			bName := b.Patch.Value.AsMap()["name"]
-			if aName != nil && bName != nil {
-				// EnvoyFilter for ECDS
-				as, _ := aName.(string)
-				bs, _ := bName.(string)
-				return as < bs
-			} else if aName != nil {
-				return true
-			} else if bName != nil {
-				return false
-			}
-			aVhost := a.Match.GetRouteConfiguration().GetVhost()
-			bVhost := b.Match.GetRouteConfiguration().GetVhost()
-			if aVhost.Name != bVhost.Name {
-				return aVhost.Name < bVhost.Name
-			}
-			return aVhost.GetRoute().Name < bVhost.GetRoute().Name
-		})
+		if strings.HasPrefix(ef.Name, "htnn-h-") {
+			// Sort here to avoid EnvoyFilter change caused by the order of ConfigPatch.
+			sort.Slice(ef.Spec.ConfigPatches, func(i, j int) bool {
+				a := ef.Spec.ConfigPatches[i]
+				b := ef.Spec.ConfigPatches[j]
+				aVhost := a.Match.GetRouteConfiguration().GetVhost()
+				bVhost := b.Match.GetRouteConfiguration().GetVhost()
+				if aVhost.Name != bVhost.Name {
+					return aVhost.Name < bVhost.Name
+				}
+				return aVhost.GetRoute().Name < bVhost.GetRoute().Name
+			})
+		}
+		// For EnvoyFilter to LDS, we need to keep the original filter order
+
 		efs[key] = ef.EnvoyFilter
 	}
 

controller/internal/translation/final_state_test.go

@@ -0,0 +1,34 @@
+// Copyright The HTNN Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package translation
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEnvoyFilterNameFromLds(t *testing.T) {
+	for _, ca := range []struct {
+		input    string
+		expected string
+	}{
+		{"::_80", "htnn-lds----80"},
+	} {
+		out := envoyFilterNameFromLds(ca.input)
+		assert.Equal(t, ca.expected, out)
+		assert.True(t, validEnvoyFilterName.MatchString(out))
+	}
+}

controller/internal/translation/init_state.go

@@ -200,10 +200,6 @@ func (s *InitState) AddPolicyForIstioGateway(policy *mosniov1.FilterPolicy, gw *
 
 	for _, svr := range gw.Spec.Servers {
 		proto := mosniov1.NormalizeIstioProtocol(svr.Port.Protocol)
-		if proto != "HTTP" && proto != "HTTPS" {
-			continue
-		}
-
 		scope := PolicyScopeGateway
 		if targetRef != nil && targetRef.SectionName != nil {
 			if svr.Name != string(*targetRef.SectionName) {