Recording network traffic between clients and WCF services to a database

Developers working with WCF services might have come accross the excellent tool Fiddler - which is a HTTP web debugging proxy. In some production environments, it could be interesting to be able to log network traffic in the form of requests and responses to database.There are several approaches possible to this, one is to implement a WCF Message Inspector, which will also act as a WCF servicebehavior.

RecordingMessageBehavior

The following class implements the two interfaces IDispatchMessageInspector, IServiceBehavior. Please note that the code contains some domain-specific production code. The code can be adjusted to work on other code bases of course. The core structure of the code is the required code for recording network traffic. The code below contains some claims-specific code handling that is not required, but perhaps your code base also use some similar code?
In other words, you have to adjusted the code below to make it work with your code base, but the core structure of the code should be of general interest for WCF developers.

using System;
using System.Configuration;
using System.ServiceModel;
using System.ServiceModel.Channels;
using System.ServiceModel.Description;
using System.ServiceModel.Dispatcher;

using Microsoft.IdentityModel.Claims;
using System.Diagnostics;

namespace Hemit.OpPlan.Service.Host
{
    public class RecordingMessageBehavior : IDispatchMessageInspector, IServiceBehavior
    {

        private readonly bool _useTrafficRecording;

        private static DateTime _lastClearTime = DateTime.Now; 

        private readonly ITrafficLogManager _trafficLogManager;


        public RecordingMessageBehavior()
        {
            _trafficLogManager = new TrafficLogManager(); 

            try
            {
                if (ConfigurationManager.AppSettings[Constants.UseTrafficRecording] != null)
                {
                    _useTrafficRecording = bool.Parse(ConfigurationManager.AppSettings[Constants.UseTrafficRecording]); 
                }
            }
            catch (Exception err)
            {
                Debug.WriteLine(err.Message);
            }
        }


        #region IDispatchMessageInspector Members

        public object AfterReceiveRequest(ref Message request, IClientChannel channel, InstanceContext instanceContext)
        {
            MessageBuffer buffer = request.CreateBufferedCopy(Int32.MaxValue);
            request = buffer.CreateMessage();
            Message messageToWorkWith = buffer.CreateMessage(); 

            if (IsTrafficRecordingDeactivated())
                return request;
            InspectMessage(messageToWorkWith, true);
            return request;
        }

        public void BeforeSendReply(ref Message reply, object correlationState)
        {
            MessageBuffer buffer = reply.CreateBufferedCopy(Int32.MaxValue);
            reply = buffer.CreateMessage();
            Message messageToWorkWith = buffer.CreateMessage(); 

            if (IsTrafficRecordingDeactivated())
                return;

            InspectMessage(messageToWorkWith, false);
        }

        private bool IsTrafficRecordingDeactivated()
        {
            return !_useTrafficRecording || !IsCurrentUserLoggedIn();
        }

        #endregion

        private void InspectMessage(Message message, bool isRequest)
        {
            try
            {
                string serializedContent = message.ToString();
                string remoteIp = null;

                if (isRequest)
                {
                    try
                    {
                        RemoteEndpointMessageProperty remoteEndpoint = message.Properties[RemoteEndpointMessageProperty.Name] as RemoteEndpointMessageProperty;
                        if (remoteEndpoint != null) remoteIp = remoteEndpoint.Address + ":" + remoteEndpoint.Port;
                    }
                    catch (Exception err)
                    {
                        Debug.WriteLine(err.Message);
                    }
                }

                var trafficLogData = new TrafficLogItemDataContract
                {
                    IsRequest = isRequest,
                    RequestIP = remoteIp,
                    Recorded = DateTime.Now,
                    SoapMessage = serializedContent,
                    ServiceMethod = GetSoapActionPart(message.Headers.Action, 1),
                    ServiceName = GetSoapActionPart(message.Headers.Action, 2)
                };

                _trafficLogManager.InsertTrafficLogItem(trafficLogData, ResolveCurrentClaimsIdentity());

                if (DateTime.Now.Subtract(_lastClearTime).TotalHours > 1)
                {
                    _trafficLogManager.ClearOldTrafficLog(ResolveCurrentClaimsIdentity()); //check if clearing old traffic logs older than an hour 
                    _lastClearTime = DateTime.Now; 
                }
            }
            catch (Exception ex)
            {
                InterfaceBinding.GetInstance<ILog>().WriteError(ex.Message, ex);

            }
        }

        private string GetSoapActionPart(string soapAction, int rightOffset)
        {
            if (string.IsNullOrEmpty(soapAction) || !soapAction.Contains("/"))
                return soapAction;

            string[] soapParts = soapAction.Split('/');
            int soapPartCount = soapParts.Length;
            if (rightOffset >= soapPartCount)
                return soapAction;
            else
                return soapParts[soapPartCount - rightOffset]; 
        }

        private IClaimsIdentity ResolveCurrentClaimsIdentity()
        {
            if (ServiceSecurityContext.Current != null && ServiceSecurityContext.Current.AuthorizationContext != null)
            {
                var claimsIdentity = ServiceSecurityContext.Current.PrimaryIdentity as IClaimsIdentity; //Retrieval of current claims identity from WCF ServiceSecurityContext
                return claimsIdentity;
            }
            return null;
        }

        private bool IsCurrentUserLoggedIn()
        {
            IClaimsIdentity claimsIdentity = ResolveCurrentClaimsIdentity();
            if (claimsIdentity == null || claimsIdentity.Claims == null || claimsIdentity.Claims.Count == 0 
                || claimsIdentity.FindClaim(WellKnownClaims.AuthorizedForOrganizationalUnit) == null)
                return false;
            return true;
        }

        #region IServiceBehavior Members

        public void AddBindingParameters(ServiceDescription serviceDescription, ServiceHostBase serviceHostBase, System.Collections.ObjectModel.Collection<ServiceEndpoint> endpoints, BindingParameterCollection bindingParameters)
        {
        }

        public void ApplyDispatchBehavior(ServiceDescription serviceDescription, ServiceHostBase serviceHostBase)
        {
            foreach (var channelDispatcherBase in serviceHostBase.ChannelDispatchers)
            {
                var chdisp = channelDispatcherBase as ChannelDispatcher;
                if (chdisp == null)
                    continue; 

                foreach (var endpoint in chdisp.Endpoints)
                {
                    endpoint.DispatchRuntime.MessageInspectors.Add(new RecordingMessageBehavior()); 
                }
            }
        }

        public void Validate(ServiceDescription serviceDescription, ServiceHostBase serviceHostBase)
        {
        }

        #endregion
    }
}

To turn on and off this WCF recording message behavior, toggle the app setting UseTrafficRecording to true in web.config:

In the code above, the first thing to do in the two methods AfterReceiveRequest and BeforeSendReply, is to clone the messages. This is because WCF messages are just like messages in MSMQ message queues - they can be consumed once. Actually we create a copy of a copy here to avoid errors.

   public object AfterReceiveRequest(ref Message request, IClientChannel channel, InstanceContext instanceContext)
        {

            MessageBuffer buffer = request.CreateBufferedCopy(Int32.MaxValue);
            request = buffer.CreateMessage();
            Message messageToWorkWith = buffer.CreateMessage(); 

            if (IsTrafficRecordingDeactivated())
                return request;
            InspectMessage(messageToWorkWith, true);
            return request;
        }

        public void BeforeSendReply(ref Message reply, object correlationState)
        {
 
            MessageBuffer buffer = reply.CreateBufferedCopy(Int32.MaxValue);
            reply = buffer.CreateMessage();
            Message messageToWorkWith = buffer.CreateMessage(); 


            if (IsTrafficRecordingDeactivated())
                return;

            InspectMessage(messageToWorkWith, false);
        }

Retrieving the serialized SOAP XML contents of the WCF messsage is easy, just running the ToString() method on the Message object. In addition, the code above requires some data access layer logic that will persist the data to database. Using EntityFramework and SQL Server her, logging is performed to the table TrafficLog. Please note that the code below will retain 24 hours of network traffic. In a production environment, database will fill up quickly if the traffic and intensity of data transfer in the form of request and response from and to the clients by the WCF services, will quickly grow into Gigabytes (GB). Usually you only want to record traffic when security demands this, or for error tracking and diagnostics.

using System;
using System.Collections.Generic;
using System.Linq;

using Microsoft.IdentityModel.Claims;


namespace Hemit.OpPlan.Service.Implementation
{

    public class TrafficLogManager : ITrafficLogManager
    {

        public List<TrafficLogItemDataContract> GetTrafficLog(DateTime startWindow, DateTime endWindow, IClaimsIdentity claimsIdentity)
        {
            using (var dbContext = ObjectContextManager.DbContextFromClaimsPrincipal(claimsIdentity))
            {
                var trafficLogs = dbContext.TrafficLogs.Where(tl => tl.Recorded >= startWindow && tl.Recorded <= endWindow).ToList();
                return trafficLogs.ForEach(tl =>
                {
                    return CreateTrafficLogDataContract(tl);
                });
            }
        }

        public void ClearOldTrafficLog(IClaimsIdentity claimsIdentity)
        {
            AggregateExceptionExtensions.CallActionLogAggregateException(() =>
            {
                System.Threading.ThreadPool.QueueUserWorkItem((object state) =>
                {
                    try
                    {
                        //Run on new thread 
                        using (var dbContext = ObjectContextManager.DbContextFromClaimsPrincipal(claimsIdentity))
                        {
                            //Use your own DbContext or ObjectContext here (EF)
                            DateTime obsoleteLogDate = DateTime.Now.Date.AddDays(-1);

                            var trafficLogs = dbContext.TrafficLogs.Where(tl => tl.Recorded <= obsoleteLogDate).ToList();
                            foreach (var tl in trafficLogs)
                            {
                                dbContext.TrafficLogs.DeleteObject(tl);
                            }
                            dbContext.SaveChanges();
                        }
                    }
                    catch (Exception err)
                    {
                        InterfaceBinding.GetInstance<ILog>().WriteError(err.Message); 
                    }
                });
            });
        }

        public void InsertTrafficLogItem(TrafficLogItemDataContract trafficLogItemdata, IClaimsIdentity claimsIdentity)
        {

            //string defaultConnectionString =  ObjectContextManager.GetConnectionString();

            using (var dbContext = ObjectContextManager.DbContextFromClaimsPrincipal(claimsIdentity))
            {
                var trafficLog = new TrafficLog();
                MapTrafficLogFromDataContract(trafficLogItemdata, trafficLog);
                dbContext.TrafficLogs.AddObject(trafficLog);
                dbContext.SaveChanges(); 
            }
        }

        private static void MapTrafficLogFromDataContract(TrafficLogItemDataContract trafficLogItem, TrafficLog trafficLog)
        {
            trafficLog.Recorded = trafficLogItem.Recorded;
            trafficLog.RequestIP = trafficLogItem.RequestIP;
            trafficLog.ServiceName = trafficLogItem.ServiceName;
            trafficLog.ServiceMethod = trafficLogItem.ServiceMethod;
            trafficLog.SoapMessage = trafficLogItem.SoapMessage;
            trafficLog.TrafficLogId = trafficLogItem.TrafficLogId;
            trafficLog.IsRequest = trafficLogItem.IsRequest;
        }

        private static TrafficLogItemDataContract CreateTrafficLogDataContract(TrafficLog trafficLog)
        {
            return new TrafficLogItemDataContract
            {
                IsRequest = trafficLog.IsRequest,
                Recorded = trafficLog.Recorded,
                ServiceMethod = trafficLog.ServiceMethod,
                ServiceName = trafficLog.ServiceName,
                SoapMessage = trafficLog.SoapMessage,
                TrafficLogId = trafficLog.TrafficLogId
            };
        }

    }    
   
}

In addition, we need to add the WCF message inspector to our service. I use a ServiceHostFactory and a class implementing ServiceHost class and override the OnOpening method:

        protected override void OnOpening()
        {
            ..


            if (Description.Behaviors.Find<RecordingMessageBehavior>() == null)
                Description.Behaviors.Add(new RecordingMessageBehavior()); 


            base.OnOpening();
        }

Here we register the service behavior for each WCF service to intercept the network traffic, clone the messages as required and log the contents to a database table, effectively gaining a way to get an overview of the requests and responses and to perform security audits, inspections and logging capabilities. You will need to do some adjustments to the code above to make it work against your codebase, but I hope this article is of general interest. The capability of recording WCF network traffic between services and clients is a very powerful feature giving the developer teams more control of their running environments. In addition, this is an important security boost. However, note that in many production environments, huge amounts of data will accumulate. Therefore, this implementation will clear the contents of the traffic log table every 24 hours. The following SQL query is a very convenient one to display data in SQL Management Studio. An XML column can be displayed and read as a documents conveniently.

select top 100 convert(xml, SoapMessage) as Payload, * from trafficlog

Obviously adjusting the Recorded datestamp column critera will pinpoint the returned data from this table more effectively. Retrieving and analysing the data might be problematic, if the amount of data reaches several gigabytes. Here is a powershell to retrieve data in a chunked fashion. Here, the data is being retrieved as 1-minute sized segments, which in a specific case chunked data up into 40 MB sized XML-files. This makes it possible to search the data using a text editor.

Powershell for retrieving traffic log data contents in a chunked fashion

#Traffic Log Bulk Copy Util

$databaseName='MYDATABASE_INSTANCENAME'
$databaseServer='MYDATABASE_SERVER'
$dateStamp = '130415'
$starthours = 12
$minuteSegmentSize = 1
$minuteSegmentNumbers = 160
$dateToInvestigate = '13.04.2015 00:00:00'
$outputFolder='SOMESHARE_UNC_PATH'

Write-Host "Traffic Log Request Bulk Output Logger"


for($i=0; $i -le $minuteSegmentNumbers; $i++){
 $dt = [datetime]::ParseExact($dateToInvestigate, "dd.MM.yyyy hh:mm:ss", $null)
 $hoursToAdd = $i / 60
 $minutesToAdd = $i % 60
 $dt = $dt.AddHours($hoursToAdd + $starthours).AddMinutes($minutesToAdd)
 $dtEnd = $dt.AddMinutes($minuteSegmentSize).AddSeconds(59)
 $startWindow = $dt.ToString("yyyy-MM-dd HH:mm:ss")
 $endWindow = $dtEnd.ToString("yyyy-MM-dd HH:mm:ss")
 $destinationFile = Join-Path -Path $outputFolder $ChildPath 
 $destinationFile += "trafficLog_$dateStamp_bulkcopy_" + $i + ".xml"
 Write-Host $("StartWindow: " + $startWindow + " EndWindow: " + $endWindow + "Destination file: " + $destinationFile)


 $bcpCmd = "bcp ""SELECT SoapMessage FROM TrafficLog WHERE IsRequest=1 AND Recorded >= '$startWindow' and Recorded <= '$endWindow'"" queryout $destinationFile -S $databaseServer -d $databaseName -c -t"","" -r""\n"" -T -x"

  Write-Host $("Bulk copy cmd: ")
  Write-Host $bcpCmd
  Write-Host "GO!"

  Invoke-Expression $bcpCmd

  Write-Host "DONE!"
 
}

By following this article, you should be now ready to write your WCF message inspector and record network traffic to your database, for increased security boost, logging and security audit capabilities and better overview of the raw data involved in the form of SOAP XML. Please read my previous article, if you want to gain insight into how to compress SOAP XML and sending it as binary data (byte arrays), even with GZip compression! Happy coding!

Share this article on LinkedIn.